feature/basic-model-setup #3

Merged
s495727 merged 9 commits from feature/basic-model-setup into main 2024-05-11 20:00:07 +02:00
13 changed files with 176 additions and 1 deletions
Showing only changes of commit e75075c141 - Show all commits

2
.gitignore vendored
View File

@ -1,2 +1,4 @@
data data
archive.zip archive.zip
.ipynb_checkpoints
__pycache__

1
src/.python-version Normal file
View File

@ -0,0 +1 @@
3.10.12

41
src/Dockerfile Normal file
View File

@ -0,0 +1,41 @@
FROM ubuntu:22.04
# Packages
RUN apt-get update && apt-get upgrade && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
curl liblzma-dev python-tk python3-tk tk-dev libssl-dev libffi-dev libncurses5-dev zlib1g zlib1g-dev \
libreadline-dev libbz2-dev libsqlite3-dev make gcc curl git-all wget python3-openssl gnupg2
# Setup CUDA
RUN apt-key del 7fa2af80 && \
wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-wsl-ubuntu.pin && \
mv cuda-wsl-ubuntu.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
wget https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
dpkg -i cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
cp /var/cuda-repo-wsl-ubuntu-12-2-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \
apt-get update && \
apt-get -y install cuda-toolkit-12-2
# Pyenv
ENV PYENV_ROOT="$HOME/.pyenv"
ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/versions/3.10.12/bin:$PATH"
RUN curl https://pyenv.run | bash
RUN pyenv install 3.10.12 && \
pyenv global 3.10.12 && \
echo 'eval "$(pyenv init --path)"' >> ~/.bashrc && \
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc
SHELL ["/bin/bash", "-c"]
WORKDIR /app
ADD ./requirements.txt /app/requirements.txt
RUN pip install -r requirements.txt
ENV CUDNN_PATH="/.pyenv/versions/3.10.12/lib/python3.10/site-packages/nvidia/cudnn/"
ENV LD_LIBRARY_PATH="$CUDNN_PATH/lib":"/usr/local/cuda-12.2/lib64"
ENV PATH="$PATH":"/usr/local/cuda-12.2/bin"
COPY . .
ARG api_key
RUN wandb login $api_key

12
src/README.md Normal file
View File

@ -0,0 +1,12 @@
# Setup
1. Install Docker on your local system
2. Build docker image and run the shell
3. Get your API key from https://wandb.ai/settings#api, docker will automatically connect to WanDB.
```bash
docker build -t gpu api_key="<wandb_api_key>" .
docker run --rm -it --gpus all --entrypoint /bin/bash gpu
```
4. To double check if tensorflow is configured properly run `python3 gpu_check.py`.

0
src/__init__.py Normal file
View File

18
src/gpu_check.py Normal file
View File

@ -0,0 +1,18 @@
try:
import tensorflow
except ImportError:
print("Tensorflow is not installed, install requied packages from requirements.txt")
exit(1)
import tensorflow
print("If you see the tensor result, then the Tensorflow is available.")
rs = tensorflow.reduce_sum(tensorflow.random.normal([1000, 1000]))
print(rs)
gpus = tensorflow.config.list_physical_devices('GPU')
if len(gpus) == 0:
print("No GPU available.")
else:
print(f"GPUs available: {len(gpus)}")
print(gpus)

7
src/main.py Normal file
View File

@ -0,0 +1,7 @@
from model.test_model import TestModel
if __name__ == "__main__":
model = TestModel()
history = model.fit()
model.save()

0
src/model/__init__.py Normal file
View File

65
src/model/test_model.py Normal file
View File

@ -0,0 +1,65 @@
import random
import tensorflow as tf
from wandb_utils.config import Config
from wandb.keras import WandbMetricsLogger
class TestModel:
def __init__(self):
self.config = Config(epoch=8, batch_size=256).config()
self.config.learning_rate = 0.01
# Define specific configuration below, they will be visible in the W&B interface
# Start of config
self.config.layer_1 = 512
self.config.activation_1 = "relu"
self.config.dropout = random.uniform(0.01, 0.80),
self.config.layer_2 = 10
self.config.activation_2 = "softmax"
self.config.optimizer = "sgd"
self.config.loss = "sparse_categorical_crossentropy"
self.config.metrics = ["accuracy"]
# End
self.model = self.__build_model()
self.__compile()
self.__load_dataset()
def __build_model(self):
return tf.keras.models.Sequential([
tf.keras.layers.Input(shape=(28,28)),
tf.keras.layers.Dense(self.config.layer_1, activation=self.config.activation_1),
tf.keras.layers.Dropout(self.config.dropout),
tf.keras.layers.Dense(self.config.layer_2, activation=self.config.activation_2)
])
def __compile(self):
self.model.compile(
optimizer=self.config.optimizer,
loss=self.config.loss,
metrics=self.config.metrics,
)
def __load_dataset(self):
mnist = tf.keras.datasets.mnist
(self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data()
self.x_train, self.x_test = self.x_train / 255.0, self.x_test / 255.0
self.x_train, self.y_train = self.x_train[::5], self.y_train[::5]
self.x_test, self.y_test = self.x_test[::20], self.y_test[::20]
def fit(self):
wandb_callbacks = [
WandbMetricsLogger(log_freq=5),
# Not supported with Keras >= 3.0.0
# WandbModelCheckpoint(filepath="models"),
]
return self.model.fit(
x=self.x_train,
y=self.y_train,
epochs=self.config.epoch,
batch_size=self.config.batch_size,
validation_data=(self.x_test, self.y_test),
callbacks=wandb_callbacks
)
def save(self):
self.model.save("test_model/final_model.keras")

7
src/requirements.txt Normal file
View File

@ -0,0 +1,7 @@
tensorflow[and-cuda]==2.16.1
tensorflow-io==0.37.0
numpy==1.26.4
opencv-python==4.9.0.80
numpy==1.26.4
wget==3.2
wandb==0.16.6

0
src/tests/__init__.py Normal file
View File

View File

22
src/wandb_utils/config.py Normal file
View File

@ -0,0 +1,22 @@
import wandb
class Config:
def __init__(self, epoch, batch_size):
self.epoch = epoch
self.batch_size = batch_size
self.run = wandb.init(
project="Detection of plant diseases",
config={
"epoch": epoch,
"batch_size": batch_size,
}
)
def config(self):
return self.run.config
def finish(self):
self.run.config.finish()