Updating project structure, dockerfile + volume setup

This commit is contained in:
s495727 2024-05-11 19:53:35 +02:00
parent 7fb9902340
commit c082a3982a
21 changed files with 257 additions and 168 deletions

View File

@ -1,41 +1,36 @@
FROM ubuntu:22.04 FROM ubuntu:22.04
# Packages # Packages
RUN apt-get update && apt-get upgrade && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get upgrade && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
curl liblzma-dev python-tk python3-tk tk-dev libssl-dev libffi-dev libncurses5-dev zlib1g zlib1g-dev \ curl liblzma-dev python-tk python3-tk tk-dev libssl-dev libffi-dev libncurses5-dev zlib1g zlib1g-dev \
libreadline-dev libbz2-dev libsqlite3-dev make gcc curl git-all wget python3-openssl gnupg2 libreadline-dev libbz2-dev libsqlite3-dev make gcc curl git-all wget python3-openssl gnupg2
# Setup CUDA # Setup CUDA
RUN apt-key del 7fa2af80 && \ RUN apt-key del 7fa2af80 && \
wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-wsl-ubuntu.pin && \ wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-wsl-ubuntu.pin && \
mv cuda-wsl-ubuntu.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \ mv cuda-wsl-ubuntu.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
wget https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \ wget https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
dpkg -i cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \ dpkg -i cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
cp /var/cuda-repo-wsl-ubuntu-12-2-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \ cp /var/cuda-repo-wsl-ubuntu-12-2-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \
apt-get update && \ apt-get update && \
apt-get -y install cuda-toolkit-12-2 apt-get -y install cuda-toolkit-12-2
# Pyenv # Pyenv
ENV PYENV_ROOT="$HOME/.pyenv" ENV PYENV_ROOT="$HOME/.pyenv"
ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/versions/3.10.12/bin:$PATH" ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/versions/3.10.12/bin:$PATH"
RUN curl https://pyenv.run | bash RUN curl https://pyenv.run | bash
RUN pyenv install 3.10.12 && \ RUN pyenv install 3.10.12 && \
pyenv global 3.10.12 && \ pyenv global 3.10.12 && \
echo 'eval "$(pyenv init --path)"' >> ~/.bashrc && \ echo 'eval "$(pyenv init --path)"' >> ~/.bashrc && \
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc
SHELL ["/bin/bash", "-c"] SHELL ["/bin/bash", "-c"]
WORKDIR /app WORKDIR /app
ADD ./requirements.txt /app/requirements.txt ADD ./requirements.txt /app/requirements.txt
RUN pip install -r requirements.txt RUN pip install -r requirements.txt
ENV CUDNN_PATH="/.pyenv/versions/3.10.12/lib/python3.10/site-packages/nvidia/cudnn/" ENV CUDNN_PATH="/.pyenv/versions/3.10.12/lib/python3.10/site-packages/nvidia/cudnn/"
ENV LD_LIBRARY_PATH="$CUDNN_PATH/lib":"/usr/local/cuda-12.2/lib64" ENV LD_LIBRARY_PATH="$CUDNN_PATH/lib":"/usr/local/cuda-12.2/lib64"
ENV PATH="$PATH":"/usr/local/cuda-12.2/bin" ENV PATH="$PATH":"/usr/local/cuda-12.2/bin"
COPY . .
ARG api_key
RUN wandb login $api_key

View File

@ -1,5 +1,6 @@
.PHONY: download-dataset resize-dataset sobel-dataset .PHONY: download-dataset resize-dataset sobel-dataset
# Use inside docker container
download-dataset: download-dataset:
python3 ./file_manager/data_manager.py --download python3 ./file_manager/data_manager.py --download
@ -7,4 +8,17 @@ resize-dataset:
python3 ./file_manager/data_manager.py --resize --shape 64 64 --source "original_dataset" python3 ./file_manager/data_manager.py --resize --shape 64 64 --source "original_dataset"
sobel-dataset: sobel-dataset:
python3 ./file_manager/data_manager.py --sobel --source "resized_dataset" python3 ./file_manager/data_manager.py --sobel --source "resized_dataset"
login:
wandb login $$(cat "$$API_KEY_SECRET")
# Outside docker
docker-run:
docker-compose run --entrypoint=/bin/bash gpu
docker-build:
docker-compose build
check-gpu:
python3 ./gpu_check.py

View File

@ -14,4 +14,24 @@
# Dokumentacja # Dokumentacja
[Link do dokumentacji](https://uam-my.sharepoint.com/personal/krzboj_st_amu_edu_pl/_layouts/15/doc.aspx?sourcedoc={dc695bbe-68d1-4947-8c29-1d008f252a3b}&action=edit) [Link do dokumentacji](https://uam-my.sharepoint.com/personal/krzboj_st_amu_edu_pl/_layouts/15/doc.aspx?sourcedoc={dc695bbe-68d1-4947-8c29-1d008f252a3b}&action=edit)
# Setup
1. Install Docker on your local system.
2. To build docker image and run the shell, use Makefile
```bash
make docker-build
make docker-run
```
3. Get your API key from https://wandb.ai/settings#api, and add the key to **secrets.txt** file.
4. After running the container
```bash
make login # to login to WanDB.
make check-gpu # to verify if GPU works
```
5. If needed, to manually run containers, run:
```bash
docker build -t gpu api_key="<wandb_api_key>" .
docker run --rm -it --gpus all --entrypoint /bin/bash gpu
```

23
compose.yaml Normal file
View File

@ -0,0 +1,23 @@
services:
gpu:
image: gpu
volumes:
- .:/app
command: nvidia-smi
build:
context: .
dockerfile: Dockerfile
environment:
API_KEY_SECRET: /run/secrets/api_key_secret
secrets:
- api_key_secret
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
secrets:
api_key_secret:
file: ./secrets.txt

15
main.py Normal file
View File

@ -0,0 +1,15 @@
from model.test_model import TestModel
from pathlib import Path
from dataset.dataset import Dataset
if __name__ == "__main__":
# Loading dataset
train_dataset = Dataset(Path('data/resized_dataset/train'))
valid_dataset = Dataset(Path('data/resized_dataset/valid'))
for i in train_dataset.take(1):
print(i)
# Training model
model = TestModel()
history = model.fit()
model.save("./src/model/test_model_final.keras")

55
model/resnet_50_model.py Normal file
View File

@ -0,0 +1,55 @@
import tensorflow as tf
from wandb_utils.config import Config
from wandb.keras import WandbMetricsLogger
class Resnet50Model:
def __init__(self):
self.config = Config(epoch=8, batch_size=64).config()
self.config.learning_rate = 0.01
# Define specific configuration below, they will be visible in the W&B interface
# Start of config
self.config.optimizer = "sgd"
self.config.loss = "sparse_categorical_crossentropy"
self.config.metrics = ["accuracy"]
# End
self.model = self.__build_model()
self.__compile()
self.__load_dataset()
def __build_model(self):
return tf.keras.applications.ResNet50(
input_shape=(224, 224, 3), include_top=False, weights='imagenet'
# output - odblokować ostatnią warstwę freeze
# zobaczyc czy dziala to by default, czy wewn. warstwy są frozen, i czy ost. jest dynamiczna
)
def __compile(self):
self.model.compile(
optimizer=self.config.optimizer,
loss=self.config.loss,
metrics=self.config.metrics,
)
def __load_dataset(self):
(self.x_train, self.y_train), (self.x_test, self.y_test) = tf.keras.datasets.cifar10.load_data()
self.x_train = self.x_train.astype('float32') / 255.0
self.x_test = self.x_test.astype('float32') / 255.0
def fit(self):
wandb_callbacks = [
WandbMetricsLogger(log_freq=5),
# Not supported with Keras >= 3.0.0
# WandbModelCheckpoint(filepath="models"),
]
return self.model.fit(
x=self.x_train,
y=self.y_train,
epochs=self.config.epoch,
batch_size=self.config.batch_size,
callbacks=wandb_callbacks
)
def save(self, filepath):
self.model.save(filepath)

View File

@ -1,65 +1,65 @@
import random import random
import tensorflow as tf import tensorflow as tf
from wandb_utils.config import Config from wandb_utils.config import Config
from wandb.keras import WandbMetricsLogger from wandb.keras import WandbMetricsLogger
class TestModel: class TestModel:
def __init__(self): def __init__(self):
self.config = Config(epoch=8, batch_size=256).config() self.config = Config(epoch=8, batch_size=256).config()
self.config.learning_rate = 0.01 self.config.learning_rate = 0.01
# Define specific configuration below, they will be visible in the W&B interface # Define specific configuration below, they will be visible in the W&B interface
# Start of config # Start of config
self.config.layer_1 = 512 self.config.layer_1 = 512
self.config.activation_1 = "relu" self.config.activation_1 = "relu"
self.config.dropout = random.uniform(0.01, 0.80) self.config.dropout = random.uniform(0.01, 0.80)
self.config.layer_2 = 10 self.config.layer_2 = 10
self.config.activation_2 = "softmax" self.config.activation_2 = "softmax"
self.config.optimizer = "sgd" self.config.optimizer = "sgd"
self.config.loss = "sparse_categorical_crossentropy" self.config.loss = "sparse_categorical_crossentropy"
self.config.metrics = ["accuracy"] self.config.metrics = ["accuracy"]
# End # End
self.model = self.__build_model() self.model = self.__build_model()
self.__compile() self.__compile()
self.__load_dataset() self.__load_dataset()
def __build_model(self): def __build_model(self):
return tf.keras.models.Sequential([ return tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(self.config.layer_1, activation=self.config.activation_1), tf.keras.layers.Dense(self.config.layer_1, activation=self.config.activation_1),
tf.keras.layers.Dropout(self.config.dropout), tf.keras.layers.Dropout(self.config.dropout),
tf.keras.layers.Dense(self.config.layer_2, activation=self.config.activation_2) tf.keras.layers.Dense(self.config.layer_2, activation=self.config.activation_2)
]) ])
def __compile(self): def __compile(self):
self.model.compile( self.model.compile(
optimizer=self.config.optimizer, optimizer=self.config.optimizer,
loss=self.config.loss, loss=self.config.loss,
metrics=self.config.metrics, metrics=self.config.metrics,
) )
def __load_dataset(self): def __load_dataset(self):
mnist = tf.keras.datasets.mnist mnist = tf.keras.datasets.mnist
(self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data() (self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data()
self.x_train, self.x_test = self.x_train / 255.0, self.x_test / 255.0 self.x_train, self.x_test = self.x_train / 255.0, self.x_test / 255.0
self.x_train, self.y_train = self.x_train[::5], self.y_train[::5] self.x_train, self.y_train = self.x_train[::5], self.y_train[::5]
self.x_test, self.y_test = self.x_test[::20], self.y_test[::20] self.x_test, self.y_test = self.x_test[::20], self.y_test[::20]
def fit(self): def fit(self):
wandb_callbacks = [ wandb_callbacks = [
WandbMetricsLogger(log_freq=5), WandbMetricsLogger(log_freq=5),
# Not supported with Keras >= 3.0.0 # Not supported with Keras >= 3.0.0
# WandbModelCheckpoint(filepath="models"), # WandbModelCheckpoint(filepath="models"),
] ]
return self.model.fit( return self.model.fit(
x=self.x_train, x=self.x_train,
y=self.y_train, y=self.y_train,
epochs=self.config.epoch, epochs=self.config.epoch,
batch_size=self.config.batch_size, batch_size=self.config.batch_size,
validation_data=(self.x_test, self.y_test), validation_data=(self.x_test, self.y_test),
callbacks=wandb_callbacks callbacks=wandb_callbacks
) )
def save(self, filepath): def save(self, filepath):
self.model.save(filepath) self.model.save(filepath)

Binary file not shown.

View File

@ -1,4 +1,7 @@
tensorflow==2.16.1 tensorflow[and-cuda]==2.16.1
tensorflow-io==0.37.0
numpy==1.26.4 numpy==1.26.4
opencv-python==4.9.0.80 opencv-python==4.9.0.80
numpy==1.26.4
wget==3.2 wget==3.2
wandb==0.16.6

1
secrets.txt Normal file
View File

@ -0,0 +1 @@
FILL IN

View File

@ -1 +0,0 @@
3.10.12

View File

@ -1,12 +0,0 @@
# Setup
1. Install Docker on your local system
2. Build docker image and run the shell
3. Get your API key from https://wandb.ai/settings#api, docker will automatically connect to WanDB.
```bash
docker build -t gpu api_key="<wandb_api_key>" .
docker run --rm -it --gpus all --entrypoint /bin/bash gpu
```
4. To double check if tensorflow is configured properly run `python3 gpu_check.py`.

View File

@ -1,7 +0,0 @@
from model.test_model import TestModel
if __name__ == "__main__":
model = TestModel()
history = model.fit()
model.save("model/test_model_final.keras")

View File

@ -1,7 +0,0 @@
tensorflow[and-cuda]==2.16.1
tensorflow-io==0.37.0
numpy==1.26.4
opencv-python==4.9.0.80
numpy==1.26.4
wget==3.2
wandb==0.16.6

View File

10
test.py
View File

@ -1,10 +0,0 @@
from pathlib import Path
from dataset.dataset import Dataset
train_dataset = Dataset(Path('data/resized_dataset/train'))
valid_dataset = Dataset(Path('data/resized_dataset/valid'))
for i in train_dataset.take(1):
print(i)

View File

@ -1,22 +1,22 @@
import wandb import wandb
class Config: class Config:
def __init__(self, epoch, batch_size): def __init__(self, epoch, batch_size):
self.epoch = epoch self.epoch = epoch
self.batch_size = batch_size self.batch_size = batch_size
self.run = wandb.init( self.run = wandb.init(
project="Detection of plant diseases", project="Detection of plant diseases",
config={ config={
"epoch": epoch, "epoch": epoch,
"batch_size": batch_size, "batch_size": batch_size,
} }
) )
def config(self): def config(self):
return self.run.config return self.run.config
def finish(self): def finish(self):
self.run.config.finish() self.run.config.finish()