Updating project structure, dockerfile + volume setup

.gitignore

@@ -1,7 +1,6 @@
 .ipynb_checkpoints
 data/
 *.zip
-
 # https://github.com/microsoft/vscode-python/blob/main/.gitignore
 .DS_Store
 .huskyrc.json

Dockerfile

@@ -1,41 +1,36 @@
-FROM ubuntu:22.04
-
-# Packages
-RUN apt-get update && apt-get upgrade && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 
-    curl liblzma-dev python-tk python3-tk tk-dev libssl-dev libffi-dev libncurses5-dev zlib1g zlib1g-dev \
-    libreadline-dev libbz2-dev libsqlite3-dev make gcc curl git-all wget python3-openssl gnupg2
-
-# Setup CUDA
-RUN apt-key del 7fa2af80 && \
-    wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-wsl-ubuntu.pin && \
-    mv cuda-wsl-ubuntu.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
-    wget https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
-    dpkg -i cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
-    cp /var/cuda-repo-wsl-ubuntu-12-2-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \
-    apt-get update && \
-    apt-get -y install cuda-toolkit-12-2
-
-# Pyenv
-ENV PYENV_ROOT="$HOME/.pyenv"
-ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/versions/3.10.12/bin:$PATH"
-
-RUN curl https://pyenv.run | bash
-RUN pyenv install 3.10.12 && \
-    pyenv global 3.10.12 && \
-    echo 'eval "$(pyenv init --path)"' >> ~/.bashrc && \
-    echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc
-
-SHELL ["/bin/bash", "-c"]
-
-WORKDIR /app
-ADD ./requirements.txt /app/requirements.txt
-RUN pip install -r requirements.txt
-
-ENV CUDNN_PATH="/.pyenv/versions/3.10.12/lib/python3.10/site-packages/nvidia/cudnn/"
-ENV LD_LIBRARY_PATH="$CUDNN_PATH/lib":"/usr/local/cuda-12.2/lib64"
-ENV PATH="$PATH":"/usr/local/cuda-12.2/bin"
-
-COPY . .
-
-ARG api_key
-RUN wandb login $api_key
+FROM ubuntu:22.04
+
+# Packages
+RUN apt-get update && apt-get upgrade && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 
+    curl liblzma-dev python-tk python3-tk tk-dev libssl-dev libffi-dev libncurses5-dev zlib1g zlib1g-dev \
+    libreadline-dev libbz2-dev libsqlite3-dev make gcc curl git-all wget python3-openssl gnupg2
+
+# Setup CUDA
+RUN apt-key del 7fa2af80 && \
+    wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-wsl-ubuntu.pin && \
+    mv cuda-wsl-ubuntu.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
+    wget https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
+    dpkg -i cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
+    cp /var/cuda-repo-wsl-ubuntu-12-2-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \
+    apt-get update && \
+    apt-get -y install cuda-toolkit-12-2
+
+# Pyenv
+ENV PYENV_ROOT="$HOME/.pyenv"
+ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/versions/3.10.12/bin:$PATH"
+
+RUN curl https://pyenv.run | bash
+RUN pyenv install 3.10.12 && \
+    pyenv global 3.10.12 && \
+    echo 'eval "$(pyenv init --path)"' >> ~/.bashrc && \
+    echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /app
+ADD ./requirements.txt /app/requirements.txt
+RUN pip install -r requirements.txt
+
+ENV CUDNN_PATH="/.pyenv/versions/3.10.12/lib/python3.10/site-packages/nvidia/cudnn/"
+ENV LD_LIBRARY_PATH="$CUDNN_PATH/lib":"/usr/local/cuda-12.2/lib64"
+ENV PATH="$PATH":"/usr/local/cuda-12.2/bin"

Makefile

@@ -1,5 +1,6 @@
 .PHONY: download-dataset resize-dataset sobel-dataset
 
+# Use inside docker container
 download-dataset:
 	python3 ./file_manager/data_manager.py --download
 
@@ -7,4 +8,17 @@ resize-dataset:
 	python3 ./file_manager/data_manager.py --resize --shape 64 64 --source "original_dataset"
 
 sobel-dataset:
-	python3 ./file_manager/data_manager.py --sobel --source "resized_dataset"
+	python3 ./file_manager/data_manager.py --sobel --source "resized_dataset"
+
+login:
+	wandb login $$(cat "$$API_KEY_SECRET")
+
+# Outside docker
+docker-run:
+	docker-compose run --entrypoint=/bin/bash gpu
+
+docker-build:
+	docker-compose build
+
+check-gpu:
+	python3 ./gpu_check.py

README.md

@@ -14,4 +14,24 @@
 
 # Dokumentacja
 
-[Link do dokumentacji](https://uam-my.sharepoint.com/personal/krzboj_st_amu_edu_pl/_layouts/15/doc.aspx?sourcedoc={dc695bbe-68d1-4947-8c29-1d008f252a3b}&action=edit)
+[Link do dokumentacji](https://uam-my.sharepoint.com/personal/krzboj_st_amu_edu_pl/_layouts/15/doc.aspx?sourcedoc={dc695bbe-68d1-4947-8c29-1d008f252a3b}&action=edit)
+
+# Setup
+
+1. Install Docker on your local system.
+2. To build docker image and run the shell, use Makefile
+```bash
+make docker-build
+make docker-run
+```
+3. Get your API key from https://wandb.ai/settings#api, and add the key to **secrets.txt** file.
+4. After running the container
+```bash
+make login # to login to WanDB.
+make check-gpu # to verify if GPU works
+```
+5. If needed, to manually run containers, run:
+```bash
+docker build -t gpu api_key="<wandb_api_key>" .
+docker run --rm -it --gpus all --entrypoint /bin/bash gpu
+```

compose.yaml

@@ -0,0 +1,23 @@
+services:
+  gpu:
+    image: gpu
+    volumes:
+      - .:/app
+    command: nvidia-smi
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      API_KEY_SECRET: /run/secrets/api_key_secret
+    secrets:
+      - api_key_secret
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+secrets:
+  api_key_secret:
+    file: ./secrets.txt

main.py

@@ -0,0 +1,15 @@
+from model.test_model import TestModel
+from pathlib import Path
+from dataset.dataset import Dataset
+
+if __name__ == "__main__":
+    # Loading dataset
+    train_dataset = Dataset(Path('data/resized_dataset/train'))
+    valid_dataset = Dataset(Path('data/resized_dataset/valid'))
+    for i in train_dataset.take(1):
+        print(i)
+
+    # Training model
+    model = TestModel()
+    history = model.fit()
+    model.save("./src/model/test_model_final.keras")

model/resnet_50_model.py

@@ -0,0 +1,55 @@
+import tensorflow as tf
+
+from wandb_utils.config import Config
+from wandb.keras import WandbMetricsLogger
+
+
+class Resnet50Model:
+    def __init__(self):
+        self.config = Config(epoch=8, batch_size=64).config()
+        self.config.learning_rate = 0.01
+        # Define specific configuration below, they will be visible in the W&B interface
+        # Start of config
+        self.config.optimizer = "sgd"
+        self.config.loss = "sparse_categorical_crossentropy"
+        self.config.metrics = ["accuracy"]
+        # End
+        self.model = self.__build_model()
+        self.__compile()
+        self.__load_dataset()
+
+    def __build_model(self):
+        return tf.keras.applications.ResNet50(
+			input_shape=(224, 224, 3), include_top=False, weights='imagenet'
+            # output - odblokować ostatnią warstwę freeze
+            # zobaczyc czy dziala to by default, czy wewn. warstwy są frozen, i czy ost. jest dynamiczna
+		)
+    
+    def __compile(self):
+        self.model.compile(
+            optimizer=self.config.optimizer,
+            loss=self.config.loss,
+            metrics=self.config.metrics,
+        )
+
+    def __load_dataset(self):
+        (self.x_train, self.y_train), (self.x_test, self.y_test) = tf.keras.datasets.cifar10.load_data()
+        self.x_train = self.x_train.astype('float32') / 255.0
+        self.x_test = self.x_test.astype('float32') / 255.0
+
+    def fit(self):
+        wandb_callbacks = [
+            WandbMetricsLogger(log_freq=5),
+            # Not supported with Keras >= 3.0.0
+            # WandbModelCheckpoint(filepath="models"), 
+        ]
+        return self.model.fit(
+            x=self.x_train,
+            y=self.y_train,
+            epochs=self.config.epoch,
+            batch_size=self.config.batch_size,
+            callbacks=wandb_callbacks
+        )
+
+    def save(self, filepath):
+        self.model.save(filepath)

model/test_model.py

@@ -1,65 +1,65 @@
-import random
-import tensorflow as tf
-
-from wandb_utils.config import Config
-from wandb.keras import WandbMetricsLogger
-
-
-class TestModel:
-    def __init__(self):
-        self.config = Config(epoch=8, batch_size=256).config()
-        self.config.learning_rate = 0.01
-        # Define specific configuration below, they will be visible in the W&B interface
-        # Start of config
-        self.config.layer_1 = 512
-        self.config.activation_1 = "relu"
-        self.config.dropout = random.uniform(0.01, 0.80)
-        self.config.layer_2 = 10
-        self.config.activation_2 = "softmax"
-        self.config.optimizer = "sgd"
-        self.config.loss = "sparse_categorical_crossentropy"
-        self.config.metrics = ["accuracy"]
-        # End
-        self.model = self.__build_model()
-        self.__compile()
-        self.__load_dataset()
-
-    def __build_model(self):
-        return tf.keras.models.Sequential([
-            tf.keras.layers.Flatten(input_shape=(28, 28)),
-            tf.keras.layers.Dense(self.config.layer_1, activation=self.config.activation_1),
-            tf.keras.layers.Dropout(self.config.dropout),
-            tf.keras.layers.Dense(self.config.layer_2, activation=self.config.activation_2)
-        ])
-    
-    def __compile(self):
-        self.model.compile(
-            optimizer=self.config.optimizer,
-            loss=self.config.loss,
-            metrics=self.config.metrics,
-        )
-    def __load_dataset(self):
-        mnist = tf.keras.datasets.mnist
-        (self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data()
-        self.x_train, self.x_test = self.x_train / 255.0, self.x_test / 255.0
-        self.x_train, self.y_train = self.x_train[::5], self.y_train[::5]
-        self.x_test, self.y_test = self.x_test[::20], self.y_test[::20]
-
-    def fit(self):
-        wandb_callbacks = [
-            WandbMetricsLogger(log_freq=5),
-            # Not supported with Keras >= 3.0.0
-            # WandbModelCheckpoint(filepath="models"), 
-        ]
-        return self.model.fit(
-            x=self.x_train,
-            y=self.y_train,
-            epochs=self.config.epoch,
-            batch_size=self.config.batch_size,
-            validation_data=(self.x_test, self.y_test),
-            callbacks=wandb_callbacks
-        )
-
-    def save(self, filepath):
-        self.model.save(filepath)
-
+import random
+import tensorflow as tf
+
+from wandb_utils.config import Config
+from wandb.keras import WandbMetricsLogger
+
+
+class TestModel:
+    def __init__(self):
+        self.config = Config(epoch=8, batch_size=256).config()
+        self.config.learning_rate = 0.01
+        # Define specific configuration below, they will be visible in the W&B interface
+        # Start of config
+        self.config.layer_1 = 512
+        self.config.activation_1 = "relu"
+        self.config.dropout = random.uniform(0.01, 0.80)
+        self.config.layer_2 = 10
+        self.config.activation_2 = "softmax"
+        self.config.optimizer = "sgd"
+        self.config.loss = "sparse_categorical_crossentropy"
+        self.config.metrics = ["accuracy"]
+        # End
+        self.model = self.__build_model()
+        self.__compile()
+        self.__load_dataset()
+
+    def __build_model(self):
+        return tf.keras.models.Sequential([
+            tf.keras.layers.Flatten(input_shape=(28, 28)),
+            tf.keras.layers.Dense(self.config.layer_1, activation=self.config.activation_1),
+            tf.keras.layers.Dropout(self.config.dropout),
+            tf.keras.layers.Dense(self.config.layer_2, activation=self.config.activation_2)
+        ])
+    
+    def __compile(self):
+        self.model.compile(
+            optimizer=self.config.optimizer,
+            loss=self.config.loss,
+            metrics=self.config.metrics,
+        )
+    def __load_dataset(self):
+        mnist = tf.keras.datasets.mnist
+        (self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data()
+        self.x_train, self.x_test = self.x_train / 255.0, self.x_test / 255.0
+        self.x_train, self.y_train = self.x_train[::5], self.y_train[::5]
+        self.x_test, self.y_test = self.x_test[::20], self.y_test[::20]
+
+    def fit(self):
+        wandb_callbacks = [
+            WandbMetricsLogger(log_freq=5),
+            # Not supported with Keras >= 3.0.0
+            # WandbModelCheckpoint(filepath="models"), 
+        ]
+        return self.model.fit(
+            x=self.x_train,
+            y=self.y_train,
+            epochs=self.config.epoch,
+            batch_size=self.config.batch_size,
+            validation_data=(self.x_test, self.y_test),
+            callbacks=wandb_callbacks
+        )
+
+    def save(self, filepath):
+        self.model.save(filepath)
+

requirements.txt

@@ -1,4 +1,7 @@
-tensorflow==2.16.1
+tensorflow[and-cuda]==2.16.1
+tensorflow-io==0.37.0
 numpy==1.26.4
 opencv-python==4.9.0.80
+numpy==1.26.4
 wget==3.2
+wandb==0.16.6

secrets.txt

@@ -0,0 +1 @@
+FILL IN

src/.python-version

@@ -1 +0,0 @@
-3.10.12

src/README.md

@@ -1,12 +0,0 @@
-# Setup
-
-1. Install Docker on your local system
-2. Build docker image and run the shell
-3. Get your API key from https://wandb.ai/settings#api, docker will automatically connect to WanDB.
-
-```bash
-docker build -t gpu api_key="<wandb_api_key>" .
-docker run --rm -it --gpus all --entrypoint /bin/bash gpu
-```
-
-4. To double check if tensorflow is configured properly run `python3 gpu_check.py`.

src/main.py

@@ -1,7 +0,0 @@
-
-from model.test_model import TestModel
-
-if __name__ == "__main__":
-    model = TestModel()
-    history = model.fit()
-    model.save("model/test_model_final.keras")

src/requirements.txt

@@ -1,7 +0,0 @@
-tensorflow[and-cuda]==2.16.1
-tensorflow-io==0.37.0
-numpy==1.26.4
-opencv-python==4.9.0.80
-numpy==1.26.4
-wget==3.2
-wandb==0.16.6

test.py

@@ -1,10 +0,0 @@
-
-from pathlib import Path
-
-from dataset.dataset import Dataset
-
-train_dataset = Dataset(Path('data/resized_dataset/train'))
-valid_dataset = Dataset(Path('data/resized_dataset/valid'))
-
-for i in train_dataset.take(1):
-    print(i)

wandb_utils/config.py

@@ -1,22 +1,22 @@
-import wandb
-
-class Config:
-    def __init__(self, epoch, batch_size):
-        self.epoch = epoch
-        self.batch_size = batch_size
-
-        self.run = wandb.init(
-            project="Detection of plant diseases",
-            config={
-                "epoch": epoch,
-                "batch_size": batch_size,
-            }
-        )
-
-    def config(self):
-        return self.run.config
-
-    def finish(self):
-        self.run.config.finish()
-
-
+import wandb
+
+class Config:
+    def __init__(self, epoch, batch_size):
+        self.epoch = epoch
+        self.batch_size = batch_size
+
+        self.run = wandb.init(
+            project="Detection of plant diseases",
+            config={
+                "epoch": epoch,
+                "batch_size": batch_size,
+            }
+        )
+
+    def config(self):
+        return self.run.config
+
+    def finish(self):
+        self.run.config.finish()
+
+