Updating project structure, dockerfile + volume setup

.gitignore

@@ -1,7 +1,6 @@
 .ipynb_checkpoints
 data/
 *.zip
-
 # https://github.com/microsoft/vscode-python/blob/main/.gitignore
 .DS_Store
 .huskyrc.json

Dockerfile

@@ -34,8 +34,3 @@ RUN pip install -r requirements.txt
 ENV CUDNN_PATH="/.pyenv/versions/3.10.12/lib/python3.10/site-packages/nvidia/cudnn/"
 ENV LD_LIBRARY_PATH="$CUDNN_PATH/lib":"/usr/local/cuda-12.2/lib64"
 ENV PATH="$PATH":"/usr/local/cuda-12.2/bin"
-
-COPY . .
-
-ARG api_key
-RUN wandb login $api_key

Makefile

@@ -1,5 +1,6 @@
 .PHONY: download-dataset resize-dataset sobel-dataset
 
+# Use inside docker container
 download-dataset:
 	python3 ./file_manager/data_manager.py --download
 
@@ -8,3 +9,16 @@ resize-dataset:
 
 sobel-dataset:
 	python3 ./file_manager/data_manager.py --sobel --source "resized_dataset"
+
+login:
+	wandb login $$(cat "$$API_KEY_SECRET")
+
+# Outside docker
+docker-run:
+	docker-compose run --entrypoint=/bin/bash gpu
+
+docker-build:
+	docker-compose build
+
+check-gpu:
+	python3 ./gpu_check.py

README.md

@@ -15,3 +15,23 @@
 # Dokumentacja
 
 [Link do dokumentacji](https://uam-my.sharepoint.com/personal/krzboj_st_amu_edu_pl/_layouts/15/doc.aspx?sourcedoc={dc695bbe-68d1-4947-8c29-1d008f252a3b}&action=edit)
+
+# Setup
+
+1. Install Docker on your local system.
+2. To build docker image and run the shell, use Makefile
+```bash
+make docker-build
+make docker-run
+```
+3. Get your API key from https://wandb.ai/settings#api, and add the key to **secrets.txt** file.
+4. After running the container
+```bash
+make login # to login to WanDB.
+make check-gpu # to verify if GPU works
+```
+5. If needed, to manually run containers, run:
+```bash
+docker build -t gpu api_key="<wandb_api_key>" .
+docker run --rm -it --gpus all --entrypoint /bin/bash gpu
+```

compose.yaml

@@ -0,0 +1,23 @@
+services:
+  gpu:
+    image: gpu
+    volumes:
+      - .:/app
+    command: nvidia-smi
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      API_KEY_SECRET: /run/secrets/api_key_secret
+    secrets:
+      - api_key_secret
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+secrets:
+  api_key_secret:
+    file: ./secrets.txt

main.py

@@ -0,0 +1,15 @@
+from model.test_model import TestModel
+from pathlib import Path
+from dataset.dataset import Dataset
+
+if __name__ == "__main__":
+    # Loading dataset
+    train_dataset = Dataset(Path('data/resized_dataset/train'))
+    valid_dataset = Dataset(Path('data/resized_dataset/valid'))
+    for i in train_dataset.take(1):
+        print(i)
+
+    # Training model
+    model = TestModel()
+    history = model.fit()
+    model.save("./src/model/test_model_final.keras")

model/resnet_50_model.py

@@ -0,0 +1,55 @@
+import tensorflow as tf
+
+from wandb_utils.config import Config
+from wandb.keras import WandbMetricsLogger
+
+
+class Resnet50Model:
+    def __init__(self):
+        self.config = Config(epoch=8, batch_size=64).config()
+        self.config.learning_rate = 0.01
+        # Define specific configuration below, they will be visible in the W&B interface
+        # Start of config
+        self.config.optimizer = "sgd"
+        self.config.loss = "sparse_categorical_crossentropy"
+        self.config.metrics = ["accuracy"]
+        # End
+        self.model = self.__build_model()
+        self.__compile()
+        self.__load_dataset()
+
+    def __build_model(self):
+        return tf.keras.applications.ResNet50(
+			input_shape=(224, 224, 3), include_top=False, weights='imagenet'
+            # output - odblokować ostatnią warstwę freeze
+            # zobaczyc czy dziala to by default, czy wewn. warstwy są frozen, i czy ost. jest dynamiczna
+		)
+    
+    def __compile(self):
+        self.model.compile(
+            optimizer=self.config.optimizer,
+            loss=self.config.loss,
+            metrics=self.config.metrics,
+        )
+
+    def __load_dataset(self):
+        (self.x_train, self.y_train), (self.x_test, self.y_test) = tf.keras.datasets.cifar10.load_data()
+        self.x_train = self.x_train.astype('float32') / 255.0
+        self.x_test = self.x_test.astype('float32') / 255.0
+
+    def fit(self):
+        wandb_callbacks = [
+            WandbMetricsLogger(log_freq=5),
+            # Not supported with Keras >= 3.0.0
+            # WandbModelCheckpoint(filepath="models"), 
+        ]
+        return self.model.fit(
+            x=self.x_train,
+            y=self.y_train,
+            epochs=self.config.epoch,
+            batch_size=self.config.batch_size,
+            callbacks=wandb_callbacks
+        )
+
+    def save(self, filepath):
+        self.model.save(filepath)

requirements.txt

@@ -1,4 +1,7 @@
-tensorflow==2.16.1
+tensorflow[and-cuda]==2.16.1
+tensorflow-io==0.37.0
 numpy==1.26.4
 opencv-python==4.9.0.80
+numpy==1.26.4
 wget==3.2
+wandb==0.16.6

secrets.txt

@@ -0,0 +1 @@
+FILL IN

src/.python-version

@@ -1 +0,0 @@
-3.10.12

src/README.md

@@ -1,12 +0,0 @@
-# Setup
-
-1. Install Docker on your local system
-2. Build docker image and run the shell
-3. Get your API key from https://wandb.ai/settings#api, docker will automatically connect to WanDB.
-
-```bash
-docker build -t gpu api_key="<wandb_api_key>" .
-docker run --rm -it --gpus all --entrypoint /bin/bash gpu
-```
-
-4. To double check if tensorflow is configured properly run `python3 gpu_check.py`.

src/main.py

@@ -1,7 +0,0 @@
-
-from model.test_model import TestModel
-
-if __name__ == "__main__":
-    model = TestModel()
-    history = model.fit()
-    model.save("model/test_model_final.keras")

src/requirements.txt

@@ -1,7 +0,0 @@
-tensorflow[and-cuda]==2.16.1
-tensorflow-io==0.37.0
-numpy==1.26.4
-opencv-python==4.9.0.80
-numpy==1.26.4
-wget==3.2
-wandb==0.16.6

test.py

@@ -1,10 +0,0 @@
-
-from pathlib import Path
-
-from dataset.dataset import Dataset
-
-train_dataset = Dataset(Path('data/resized_dataset/train'))
-valid_dataset = Dataset(Path('data/resized_dataset/valid'))
-
-for i in train_dataset.take(1):
-    print(i)