Updating project structure, dockerfile + volume setup
This commit is contained in:
parent
7fb9902340
commit
c082a3982a
|
|
@ -1,41 +1,36 @@
|
|||
FROM ubuntu:22.04
|
||||
|
||||
# Packages
|
||||
RUN apt-get update && apt-get upgrade && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
curl liblzma-dev python-tk python3-tk tk-dev libssl-dev libffi-dev libncurses5-dev zlib1g zlib1g-dev \
|
||||
libreadline-dev libbz2-dev libsqlite3-dev make gcc curl git-all wget python3-openssl gnupg2
|
||||
|
||||
# Setup CUDA
|
||||
RUN apt-key del 7fa2af80 && \
|
||||
wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-wsl-ubuntu.pin && \
|
||||
mv cuda-wsl-ubuntu.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
|
||||
wget https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
|
||||
dpkg -i cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
|
||||
cp /var/cuda-repo-wsl-ubuntu-12-2-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \
|
||||
apt-get update && \
|
||||
apt-get -y install cuda-toolkit-12-2
|
||||
|
||||
# Pyenv
|
||||
ENV PYENV_ROOT="$HOME/.pyenv"
|
||||
ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/versions/3.10.12/bin:$PATH"
|
||||
|
||||
RUN curl https://pyenv.run | bash
|
||||
RUN pyenv install 3.10.12 && \
|
||||
pyenv global 3.10.12 && \
|
||||
echo 'eval "$(pyenv init --path)"' >> ~/.bashrc && \
|
||||
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc
|
||||
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
WORKDIR /app
|
||||
ADD ./requirements.txt /app/requirements.txt
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
ENV CUDNN_PATH="/.pyenv/versions/3.10.12/lib/python3.10/site-packages/nvidia/cudnn/"
|
||||
ENV LD_LIBRARY_PATH="$CUDNN_PATH/lib":"/usr/local/cuda-12.2/lib64"
|
||||
ENV PATH="$PATH":"/usr/local/cuda-12.2/bin"
|
||||
|
||||
COPY . .
|
||||
|
||||
ARG api_key
|
||||
RUN wandb login $api_key
|
||||
FROM ubuntu:22.04
|
||||
|
||||
# Packages
|
||||
RUN apt-get update && apt-get upgrade && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
curl liblzma-dev python-tk python3-tk tk-dev libssl-dev libffi-dev libncurses5-dev zlib1g zlib1g-dev \
|
||||
libreadline-dev libbz2-dev libsqlite3-dev make gcc curl git-all wget python3-openssl gnupg2
|
||||
|
||||
# Setup CUDA
|
||||
RUN apt-key del 7fa2af80 && \
|
||||
wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-wsl-ubuntu.pin && \
|
||||
mv cuda-wsl-ubuntu.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
|
||||
wget https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
|
||||
dpkg -i cuda-repo-wsl-ubuntu-12-2-local_12.2.2-1_amd64.deb && \
|
||||
cp /var/cuda-repo-wsl-ubuntu-12-2-local/cuda-*-keyring.gpg /usr/share/keyrings/ && \
|
||||
apt-get update && \
|
||||
apt-get -y install cuda-toolkit-12-2
|
||||
|
||||
# Pyenv
|
||||
ENV PYENV_ROOT="$HOME/.pyenv"
|
||||
ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/versions/3.10.12/bin:$PATH"
|
||||
|
||||
RUN curl https://pyenv.run | bash
|
||||
RUN pyenv install 3.10.12 && \
|
||||
pyenv global 3.10.12 && \
|
||||
echo 'eval "$(pyenv init --path)"' >> ~/.bashrc && \
|
||||
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc
|
||||
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
WORKDIR /app
|
||||
ADD ./requirements.txt /app/requirements.txt
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
ENV CUDNN_PATH="/.pyenv/versions/3.10.12/lib/python3.10/site-packages/nvidia/cudnn/"
|
||||
ENV LD_LIBRARY_PATH="$CUDNN_PATH/lib":"/usr/local/cuda-12.2/lib64"
|
||||
ENV PATH="$PATH":"/usr/local/cuda-12.2/bin"
|
||||
16
Makefile
16
Makefile
|
|
@ -1,5 +1,6 @@
|
|||
.PHONY: download-dataset resize-dataset sobel-dataset
|
||||
|
||||
# Use inside docker container
|
||||
download-dataset:
|
||||
python3 ./file_manager/data_manager.py --download
|
||||
|
||||
|
|
@ -7,4 +8,17 @@ resize-dataset:
|
|||
python3 ./file_manager/data_manager.py --resize --shape 64 64 --source "original_dataset"
|
||||
|
||||
sobel-dataset:
|
||||
python3 ./file_manager/data_manager.py --sobel --source "resized_dataset"
|
||||
python3 ./file_manager/data_manager.py --sobel --source "resized_dataset"
|
||||
|
||||
login:
|
||||
wandb login $$(cat "$$API_KEY_SECRET")
|
||||
|
||||
# Outside docker
|
||||
docker-run:
|
||||
docker-compose run --entrypoint=/bin/bash gpu
|
||||
|
||||
docker-build:
|
||||
docker-compose build
|
||||
|
||||
check-gpu:
|
||||
python3 ./gpu_check.py
|
||||
|
|
|
|||
22
README.md
22
README.md
|
|
@ -14,4 +14,24 @@
|
|||
|
||||
# Dokumentacja
|
||||
|
||||
[Link do dokumentacji](https://uam-my.sharepoint.com/personal/krzboj_st_amu_edu_pl/_layouts/15/doc.aspx?sourcedoc={dc695bbe-68d1-4947-8c29-1d008f252a3b}&action=edit)
|
||||
[Link do dokumentacji](https://uam-my.sharepoint.com/personal/krzboj_st_amu_edu_pl/_layouts/15/doc.aspx?sourcedoc={dc695bbe-68d1-4947-8c29-1d008f252a3b}&action=edit)
|
||||
|
||||
# Setup
|
||||
|
||||
1. Install Docker on your local system.
|
||||
2. To build docker image and run the shell, use Makefile
|
||||
```bash
|
||||
make docker-build
|
||||
make docker-run
|
||||
```
|
||||
3. Get your API key from https://wandb.ai/settings#api, and add the key to **secrets.txt** file.
|
||||
4. After running the container
|
||||
```bash
|
||||
make login # to login to WanDB.
|
||||
make check-gpu # to verify if GPU works
|
||||
```
|
||||
5. If needed, to manually run containers, run:
|
||||
```bash
|
||||
docker build -t gpu api_key="<wandb_api_key>" .
|
||||
docker run --rm -it --gpus all --entrypoint /bin/bash gpu
|
||||
```
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
services:
|
||||
gpu:
|
||||
image: gpu
|
||||
volumes:
|
||||
- .:/app
|
||||
command: nvidia-smi
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
API_KEY_SECRET: /run/secrets/api_key_secret
|
||||
secrets:
|
||||
- api_key_secret
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
secrets:
|
||||
api_key_secret:
|
||||
file: ./secrets.txt
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
from model.test_model import TestModel
|
||||
from pathlib import Path
|
||||
from dataset.dataset import Dataset
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Loading dataset
|
||||
train_dataset = Dataset(Path('data/resized_dataset/train'))
|
||||
valid_dataset = Dataset(Path('data/resized_dataset/valid'))
|
||||
for i in train_dataset.take(1):
|
||||
print(i)
|
||||
|
||||
# Training model
|
||||
model = TestModel()
|
||||
history = model.fit()
|
||||
model.save("./src/model/test_model_final.keras")
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
import tensorflow as tf
|
||||
|
||||
from wandb_utils.config import Config
|
||||
from wandb.keras import WandbMetricsLogger
|
||||
|
||||
|
||||
class Resnet50Model:
|
||||
def __init__(self):
|
||||
self.config = Config(epoch=8, batch_size=64).config()
|
||||
self.config.learning_rate = 0.01
|
||||
# Define specific configuration below, they will be visible in the W&B interface
|
||||
# Start of config
|
||||
self.config.optimizer = "sgd"
|
||||
self.config.loss = "sparse_categorical_crossentropy"
|
||||
self.config.metrics = ["accuracy"]
|
||||
# End
|
||||
self.model = self.__build_model()
|
||||
self.__compile()
|
||||
self.__load_dataset()
|
||||
|
||||
def __build_model(self):
|
||||
return tf.keras.applications.ResNet50(
|
||||
input_shape=(224, 224, 3), include_top=False, weights='imagenet'
|
||||
# output - odblokować ostatnią warstwę freeze
|
||||
# zobaczyc czy dziala to by default, czy wewn. warstwy są frozen, i czy ost. jest dynamiczna
|
||||
)
|
||||
|
||||
def __compile(self):
|
||||
self.model.compile(
|
||||
optimizer=self.config.optimizer,
|
||||
loss=self.config.loss,
|
||||
metrics=self.config.metrics,
|
||||
)
|
||||
|
||||
def __load_dataset(self):
|
||||
(self.x_train, self.y_train), (self.x_test, self.y_test) = tf.keras.datasets.cifar10.load_data()
|
||||
self.x_train = self.x_train.astype('float32') / 255.0
|
||||
self.x_test = self.x_test.astype('float32') / 255.0
|
||||
|
||||
def fit(self):
|
||||
wandb_callbacks = [
|
||||
WandbMetricsLogger(log_freq=5),
|
||||
# Not supported with Keras >= 3.0.0
|
||||
# WandbModelCheckpoint(filepath="models"),
|
||||
]
|
||||
return self.model.fit(
|
||||
x=self.x_train,
|
||||
y=self.y_train,
|
||||
epochs=self.config.epoch,
|
||||
batch_size=self.config.batch_size,
|
||||
callbacks=wandb_callbacks
|
||||
)
|
||||
|
||||
def save(self, filepath):
|
||||
self.model.save(filepath)
|
||||
|
|
@ -1,65 +1,65 @@
|
|||
import random
|
||||
import tensorflow as tf
|
||||
|
||||
from wandb_utils.config import Config
|
||||
from wandb.keras import WandbMetricsLogger
|
||||
|
||||
|
||||
class TestModel:
|
||||
def __init__(self):
|
||||
self.config = Config(epoch=8, batch_size=256).config()
|
||||
self.config.learning_rate = 0.01
|
||||
# Define specific configuration below, they will be visible in the W&B interface
|
||||
# Start of config
|
||||
self.config.layer_1 = 512
|
||||
self.config.activation_1 = "relu"
|
||||
self.config.dropout = random.uniform(0.01, 0.80)
|
||||
self.config.layer_2 = 10
|
||||
self.config.activation_2 = "softmax"
|
||||
self.config.optimizer = "sgd"
|
||||
self.config.loss = "sparse_categorical_crossentropy"
|
||||
self.config.metrics = ["accuracy"]
|
||||
# End
|
||||
self.model = self.__build_model()
|
||||
self.__compile()
|
||||
self.__load_dataset()
|
||||
|
||||
def __build_model(self):
|
||||
return tf.keras.models.Sequential([
|
||||
tf.keras.layers.Flatten(input_shape=(28, 28)),
|
||||
tf.keras.layers.Dense(self.config.layer_1, activation=self.config.activation_1),
|
||||
tf.keras.layers.Dropout(self.config.dropout),
|
||||
tf.keras.layers.Dense(self.config.layer_2, activation=self.config.activation_2)
|
||||
])
|
||||
|
||||
def __compile(self):
|
||||
self.model.compile(
|
||||
optimizer=self.config.optimizer,
|
||||
loss=self.config.loss,
|
||||
metrics=self.config.metrics,
|
||||
)
|
||||
def __load_dataset(self):
|
||||
mnist = tf.keras.datasets.mnist
|
||||
(self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data()
|
||||
self.x_train, self.x_test = self.x_train / 255.0, self.x_test / 255.0
|
||||
self.x_train, self.y_train = self.x_train[::5], self.y_train[::5]
|
||||
self.x_test, self.y_test = self.x_test[::20], self.y_test[::20]
|
||||
|
||||
def fit(self):
|
||||
wandb_callbacks = [
|
||||
WandbMetricsLogger(log_freq=5),
|
||||
# Not supported with Keras >= 3.0.0
|
||||
# WandbModelCheckpoint(filepath="models"),
|
||||
]
|
||||
return self.model.fit(
|
||||
x=self.x_train,
|
||||
y=self.y_train,
|
||||
epochs=self.config.epoch,
|
||||
batch_size=self.config.batch_size,
|
||||
validation_data=(self.x_test, self.y_test),
|
||||
callbacks=wandb_callbacks
|
||||
)
|
||||
|
||||
def save(self, filepath):
|
||||
self.model.save(filepath)
|
||||
|
||||
import random
|
||||
import tensorflow as tf
|
||||
|
||||
from wandb_utils.config import Config
|
||||
from wandb.keras import WandbMetricsLogger
|
||||
|
||||
|
||||
class TestModel:
|
||||
def __init__(self):
|
||||
self.config = Config(epoch=8, batch_size=256).config()
|
||||
self.config.learning_rate = 0.01
|
||||
# Define specific configuration below, they will be visible in the W&B interface
|
||||
# Start of config
|
||||
self.config.layer_1 = 512
|
||||
self.config.activation_1 = "relu"
|
||||
self.config.dropout = random.uniform(0.01, 0.80)
|
||||
self.config.layer_2 = 10
|
||||
self.config.activation_2 = "softmax"
|
||||
self.config.optimizer = "sgd"
|
||||
self.config.loss = "sparse_categorical_crossentropy"
|
||||
self.config.metrics = ["accuracy"]
|
||||
# End
|
||||
self.model = self.__build_model()
|
||||
self.__compile()
|
||||
self.__load_dataset()
|
||||
|
||||
def __build_model(self):
|
||||
return tf.keras.models.Sequential([
|
||||
tf.keras.layers.Flatten(input_shape=(28, 28)),
|
||||
tf.keras.layers.Dense(self.config.layer_1, activation=self.config.activation_1),
|
||||
tf.keras.layers.Dropout(self.config.dropout),
|
||||
tf.keras.layers.Dense(self.config.layer_2, activation=self.config.activation_2)
|
||||
])
|
||||
|
||||
def __compile(self):
|
||||
self.model.compile(
|
||||
optimizer=self.config.optimizer,
|
||||
loss=self.config.loss,
|
||||
metrics=self.config.metrics,
|
||||
)
|
||||
def __load_dataset(self):
|
||||
mnist = tf.keras.datasets.mnist
|
||||
(self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data()
|
||||
self.x_train, self.x_test = self.x_train / 255.0, self.x_test / 255.0
|
||||
self.x_train, self.y_train = self.x_train[::5], self.y_train[::5]
|
||||
self.x_test, self.y_test = self.x_test[::20], self.y_test[::20]
|
||||
|
||||
def fit(self):
|
||||
wandb_callbacks = [
|
||||
WandbMetricsLogger(log_freq=5),
|
||||
# Not supported with Keras >= 3.0.0
|
||||
# WandbModelCheckpoint(filepath="models"),
|
||||
]
|
||||
return self.model.fit(
|
||||
x=self.x_train,
|
||||
y=self.y_train,
|
||||
epochs=self.config.epoch,
|
||||
batch_size=self.config.batch_size,
|
||||
validation_data=(self.x_test, self.y_test),
|
||||
callbacks=wandb_callbacks
|
||||
)
|
||||
|
||||
def save(self, filepath):
|
||||
self.model.save(filepath)
|
||||
|
||||
Binary file not shown.
|
|
@ -1,4 +1,7 @@
|
|||
tensorflow==2.16.1
|
||||
tensorflow[and-cuda]==2.16.1
|
||||
tensorflow-io==0.37.0
|
||||
numpy==1.26.4
|
||||
opencv-python==4.9.0.80
|
||||
numpy==1.26.4
|
||||
wget==3.2
|
||||
wandb==0.16.6
|
||||
|
|
@ -0,0 +1 @@
|
|||
FILL IN
|
||||
|
|
@ -1 +0,0 @@
|
|||
3.10.12
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
# Setup
|
||||
|
||||
1. Install Docker on your local system
|
||||
2. Build docker image and run the shell
|
||||
3. Get your API key from https://wandb.ai/settings#api, docker will automatically connect to WanDB.
|
||||
|
||||
```bash
|
||||
docker build -t gpu api_key="<wandb_api_key>" .
|
||||
docker run --rm -it --gpus all --entrypoint /bin/bash gpu
|
||||
```
|
||||
|
||||
4. To double check if tensorflow is configured properly run `python3 gpu_check.py`.
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
|
||||
from model.test_model import TestModel
|
||||
|
||||
if __name__ == "__main__":
|
||||
model = TestModel()
|
||||
history = model.fit()
|
||||
model.save("model/test_model_final.keras")
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
tensorflow[and-cuda]==2.16.1
|
||||
tensorflow-io==0.37.0
|
||||
numpy==1.26.4
|
||||
opencv-python==4.9.0.80
|
||||
numpy==1.26.4
|
||||
wget==3.2
|
||||
wandb==0.16.6
|
||||
10
test.py
10
test.py
|
|
@ -1,10 +0,0 @@
|
|||
|
||||
from pathlib import Path
|
||||
|
||||
from dataset.dataset import Dataset
|
||||
|
||||
train_dataset = Dataset(Path('data/resized_dataset/train'))
|
||||
valid_dataset = Dataset(Path('data/resized_dataset/valid'))
|
||||
|
||||
for i in train_dataset.take(1):
|
||||
print(i)
|
||||
|
|
@ -1,22 +1,22 @@
|
|||
import wandb
|
||||
|
||||
class Config:
|
||||
def __init__(self, epoch, batch_size):
|
||||
self.epoch = epoch
|
||||
self.batch_size = batch_size
|
||||
|
||||
self.run = wandb.init(
|
||||
project="Detection of plant diseases",
|
||||
config={
|
||||
"epoch": epoch,
|
||||
"batch_size": batch_size,
|
||||
}
|
||||
)
|
||||
|
||||
def config(self):
|
||||
return self.run.config
|
||||
|
||||
def finish(self):
|
||||
self.run.config.finish()
|
||||
|
||||
|
||||
import wandb
|
||||
|
||||
class Config:
|
||||
def __init__(self, epoch, batch_size):
|
||||
self.epoch = epoch
|
||||
self.batch_size = batch_size
|
||||
|
||||
self.run = wandb.init(
|
||||
project="Detection of plant diseases",
|
||||
config={
|
||||
"epoch": epoch,
|
||||
"batch_size": batch_size,
|
||||
}
|
||||
)
|
||||
|
||||
def config(self):
|
||||
return self.run.config
|
||||
|
||||
def finish(self):
|
||||
self.run.config.finish()
|
||||
|
||||
|
||||
Loading…
Reference in New Issue