From e75075c141280f6b4244cc4429715767318454ff Mon Sep 17 00:00:00 2001 From: Krzysztof Bojakowski Date: Sun, 5 May 2024 18:23:13 +0200 Subject: [PATCH] Run example with Dockerfile to run the code --- .gitignore | 4 ++- src/.python-version | 1 + src/Dockerfile | 41 +++++++++++++++++++++++ src/README.md | 12 +++++++ src/__init__.py | 0 src/gpu_check.py | 18 ++++++++++ src/main.py | 7 ++++ src/model/__init__.py | 0 src/model/test_model.py | 65 +++++++++++++++++++++++++++++++++++++ src/requirements.txt | 7 ++++ src/tests/__init__.py | 0 src/wandb_utils/__init__.py | 0 src/wandb_utils/config.py | 22 +++++++++++++ 13 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 src/.python-version create mode 100644 src/Dockerfile create mode 100644 src/README.md create mode 100644 src/__init__.py create mode 100644 src/gpu_check.py create mode 100644 src/main.py create mode 100644 src/model/__init__.py create mode 100644 src/model/test_model.py create mode 100644 src/requirements.txt create mode 100644 src/tests/__init__.py create mode 100644 src/wandb_utils/__init__.py create mode 100644 src/wandb_utils/config.py diff --git a/.gitignore b/.gitignore index 6f951ce..f47c613 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ data -archive.zip \ No newline at end of file +archive.zip +.ipynb_checkpoints +__pycache__ \ No newline at end of file diff --git a/src/.python-version b/src/.python-version new file mode 100644 index 0000000..56d91d3 --- /dev/null +++ b/src/.python-version @@ -0,0 +1 @@ +3.10.12 diff --git a/src/Dockerfile b/src/Dockerfile new file mode 100644 index 0000000..a267c7e --- /dev/null +++ b/src/Dockerfile @@ -0,0 +1,41 @@ +FROM ubuntu:22.04 + +# Packages +RUN apt-get update && apt-get upgrade && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + curl liblzma-dev python-tk python3-tk tk-dev libssl-dev libffi-dev libncurses5-dev zlib1g zlib1g-dev \ + libreadline-dev libbz2-dev libsqlite3-dev make gcc curl git-all wget python3-openssl gnupg2 + +# Setup CUDA +RUN apt-key del 7fa2af80 && \ + wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-wsl-ubuntu.pin && \ + mv cuda-wsl-ubuntu.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \ + wget https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \ + dpkg -i cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \ + cp /var/cuda-repo-wsl-ubuntu-12-2-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \ + apt-get update && \ + apt-get -y install cuda-toolkit-12-2 + +# Pyenv +ENV PYENV_ROOT="$HOME/.pyenv" +ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/versions/3.10.12/bin:$PATH" + +RUN curl https://pyenv.run | bash +RUN pyenv install 3.10.12 && \ + pyenv global 3.10.12 && \ + echo 'eval "$(pyenv init --path)"' >> ~/.bashrc && \ + echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc + +SHELL ["/bin/bash", "-c"] + +WORKDIR /app +ADD ./requirements.txt /app/requirements.txt +RUN pip install -r requirements.txt + +ENV CUDNN_PATH="/.pyenv/versions/3.10.12/lib/python3.10/site-packages/nvidia/cudnn/" +ENV LD_LIBRARY_PATH="$CUDNN_PATH/lib":"/usr/local/cuda-12.2/lib64" +ENV PATH="$PATH":"/usr/local/cuda-12.2/bin" + +COPY . . + +ARG api_key +RUN wandb login $api_key \ No newline at end of file diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..377286f --- /dev/null +++ b/src/README.md @@ -0,0 +1,12 @@ +# Setup + +1. Install Docker on your local system +2. Build docker image and run the shell +3. Get your API key from https://wandb.ai/settings#api, docker will automatically connect to WanDB. + +```bash +docker build -t gpu api_key="" . +docker run --rm -it --gpus all --entrypoint /bin/bash gpu +``` + +4. To double check if tensorflow is configured properly run `python3 gpu_check.py`. diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/gpu_check.py b/src/gpu_check.py new file mode 100644 index 0000000..282cb65 --- /dev/null +++ b/src/gpu_check.py @@ -0,0 +1,18 @@ +try: + import tensorflow +except ImportError: + print("Tensorflow is not installed, install requied packages from requirements.txt") + exit(1) + +import tensorflow + +print("If you see the tensor result, then the Tensorflow is available.") +rs = tensorflow.reduce_sum(tensorflow.random.normal([1000, 1000])) +print(rs) + +gpus = tensorflow.config.list_physical_devices('GPU') +if len(gpus) == 0: + print("No GPU available.") +else: + print(f"GPUs available: {len(gpus)}") + print(gpus) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..90c6a4c --- /dev/null +++ b/src/main.py @@ -0,0 +1,7 @@ + +from model.test_model import TestModel + +if __name__ == "__main__": + model = TestModel() + history = model.fit() + model.save() diff --git a/src/model/__init__.py b/src/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/model/test_model.py b/src/model/test_model.py new file mode 100644 index 0000000..3819942 --- /dev/null +++ b/src/model/test_model.py @@ -0,0 +1,65 @@ +import random +import tensorflow as tf + +from wandb_utils.config import Config +from wandb.keras import WandbMetricsLogger + + +class TestModel: + def __init__(self): + self.config = Config(epoch=8, batch_size=256).config() + self.config.learning_rate = 0.01 + # Define specific configuration below, they will be visible in the W&B interface + # Start of config + self.config.layer_1 = 512 + self.config.activation_1 = "relu" + self.config.dropout = random.uniform(0.01, 0.80), + self.config.layer_2 = 10 + self.config.activation_2 = "softmax" + self.config.optimizer = "sgd" + self.config.loss = "sparse_categorical_crossentropy" + self.config.metrics = ["accuracy"] + # End + self.model = self.__build_model() + self.__compile() + self.__load_dataset() + + def __build_model(self): + return tf.keras.models.Sequential([ + tf.keras.layers.Input(shape=(28,28)), + tf.keras.layers.Dense(self.config.layer_1, activation=self.config.activation_1), + tf.keras.layers.Dropout(self.config.dropout), + tf.keras.layers.Dense(self.config.layer_2, activation=self.config.activation_2) + ]) + + def __compile(self): + self.model.compile( + optimizer=self.config.optimizer, + loss=self.config.loss, + metrics=self.config.metrics, + ) + def __load_dataset(self): + mnist = tf.keras.datasets.mnist + (self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data() + self.x_train, self.x_test = self.x_train / 255.0, self.x_test / 255.0 + self.x_train, self.y_train = self.x_train[::5], self.y_train[::5] + self.x_test, self.y_test = self.x_test[::20], self.y_test[::20] + + def fit(self): + wandb_callbacks = [ + WandbMetricsLogger(log_freq=5), + # Not supported with Keras >= 3.0.0 + # WandbModelCheckpoint(filepath="models"), + ] + return self.model.fit( + x=self.x_train, + y=self.y_train, + epochs=self.config.epoch, + batch_size=self.config.batch_size, + validation_data=(self.x_test, self.y_test), + callbacks=wandb_callbacks + ) + + def save(self): + self.model.save("test_model/final_model.keras") + diff --git a/src/requirements.txt b/src/requirements.txt new file mode 100644 index 0000000..ac7eda6 --- /dev/null +++ b/src/requirements.txt @@ -0,0 +1,7 @@ +tensorflow[and-cuda]==2.16.1 +tensorflow-io==0.37.0 +numpy==1.26.4 +opencv-python==4.9.0.80 +numpy==1.26.4 +wget==3.2 +wandb==0.16.6 \ No newline at end of file diff --git a/src/tests/__init__.py b/src/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/wandb_utils/__init__.py b/src/wandb_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/wandb_utils/config.py b/src/wandb_utils/config.py new file mode 100644 index 0000000..cbd06e6 --- /dev/null +++ b/src/wandb_utils/config.py @@ -0,0 +1,22 @@ +import wandb + +class Config: + def __init__(self, epoch, batch_size): + self.epoch = epoch + self.batch_size = batch_size + + self.run = wandb.init( + project="Detection of plant diseases", + config={ + "epoch": epoch, + "batch_size": batch_size, + } + ) + + def config(self): + return self.run.config + + def finish(self): + self.run.config.finish() + +