Merge branch 'main' of git.wmi.amu.edu.pl:s495728/Detection-of-plant-diseases into feature/basic-model-setup

.gitignore

@@ -1,4 +1,55 @@
-data
-archive.zip
 .ipynb_checkpoints
+data/
+*.zip
+
+# https://github.com/microsoft/vscode-python/blob/main/.gitignore
+.DS_Store
+.huskyrc.json
+out
+log.log
+**/node_modules
+*.pyc
+*.vsix
+envVars.txt
+**/.vscode/.ropeproject/**
+**/testFiles/**/.cache/**
+*.noseids
+.nyc_output
+.vscode-test
 __pycache__
+npm-debug.log
+**/.mypy_cache/**
+!yarn.lock
+coverage/
+cucumber-report.json
+**/.vscode-test/**
+**/.vscode test/**
+**/.vscode-smoke/**
+**/.venv*/
+port.txt
+precommit.hook
+python_files/lib/**
+python_files/get-pip.py
+debug_coverage*/**
+languageServer/**
+languageServer.*/**
+bin/**
+obj/**
+.pytest_cache
+tmp/**
+.python-version
+.vs/
+test-results*.xml
+xunit-test-results.xml
+build/ci/performance/performance-results.json
+!build/
+debug*.log
+debugpy*.log
+pydevd*.log
+nodeLanguageServer/**
+nodeLanguageServer.*/**
+dist/**
+# translation files
+*.xlf
+package.nls.*.json
+l10n/

DataManager.py

@@ -1,55 +0,0 @@
-import glob
-import shutil
-import cv2
-from zipfile import ZipFile
-import os
-import wget
-
-mainPath="data/"
-pathToTrainAndValidDate = mainPath + "%s/**/*.*"
-pathToTestDataset = mainPath + "/test"
-originalDatasetName = "original dataset"
-
-class DataManager:
-
-    def downloadData(self):
-        if not os.path.isfile("archive.zip"):
-            wget.download("https://storage.googleapis.com/kaggle-data-sets/78313/182633/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20240502%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20240502T181500Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=87d0661313e358206b6e10d44f135d41e23501d601e58b1e8236ca28a82ccc434534564b45baa84c4d829dd1995ff384d51fe5dba3f543d00eb0763169fd712c6c8f91bb4f298db38a19b31b2d489798a9723a271aa4108d7b93345c5a64a7ef00b9b8f27d1d5f728e373c870f0287eb89bc747941f0aeeb4703c288059e2e07b7ece3a83114a9607276874a90d4ec96dde06fddb94a0d3af72848565661b1404e3ea248eeebf46374daada7df1f37db7d62b21b4ac90706ea64cc74200a58f35bfe379703e7691aeda9e39635b02f58a9f8399fa64b031b1a9bccd7f109d256c6f4886ef94fcdc11034d6da13c0f1d4d8b97cabdd295862a5107b587824ebe8")
-
-    def unzipData(self, fileName, pathToExtract):
-        if not os.path.exists(mainPath):
-            os.makedirs("data")    
-        ZipFile(fileName).extractall(mainPath + pathToExtract)
-        shutil.move("data/original dataset/test/test", "data", copy_function = shutil.copytree)
-        shutil.move("data/original dataset/New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)/train", "data/original dataset/train", copy_function = shutil.copytree)
-        shutil.move("data/original dataset/New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)/valid", "data/original dataset/valid", copy_function = shutil.copytree)
-        shutil.rmtree("data/original dataset/New Plant Diseases Dataset(Augmented)")
-        shutil.rmtree("data/Detection-of-plant-diseases/data/original dataset/test")
-
-    def writeImageToGivenPath(self, image, path):
-        os.makedirs(path.rsplit('/', 1)[0], exist_ok=True)
-        cv2.imwrite(path, image)
-
-    def resizeDataset(self, soruceDatasetName, width, height):
-        if not os.path.exists(mainPath + "resized dataset"):
-            for file in glob.glob(pathToTrainAndValidDate % soruceDatasetName, recursive=True):
-                pathToFile = file.replace("\\","/")
-                image = cv2.imread(pathToFile)
-                image = cv2.resize(image, (width, height))
-                newPath = pathToFile.replace(soruceDatasetName,"resized dataset")
-                self.writeImageToGivenPath(image,newPath)
-
-    def sobelx(self, soruceDatasetName):
-        if not os.path.exists(mainPath + "sobel dataset"):
-            for file in glob.glob(pathToTrainAndValidDate % soruceDatasetName, recursive=True):
-                pathToFile = file.replace("\\","/")
-                image = cv2.imread(pathToFile)
-                sobel = cv2.Sobel(image,cv2.CV_64F,1,0,ksize=5) 
-                newPath = pathToFile.replace(soruceDatasetName,"sobel dataset")
-                self.writeImageToGivenPath(sobel,newPath)
-
-dataManager = DataManager()
-dataManager.downloadData()
-dataManager.unzipData("archive.zip","original dataset")
-dataManager.resizeDataset("original dataset", 64, 64)
-dataManager.sobelx("resized dataset")

Makefile

@@ -0,0 +1,10 @@
+.PHONY: download-dataset resize-dataset sobel-dataset
+
+download-dataset:
+	python3 ./file_manager/data_manager.py --download
+
+resize-dataset:
+	python3 ./file_manager/data_manager.py --resize --shape 64 64 --source "original_dataset"
+
+sobel-dataset:
+	python3 ./file_manager/data_manager.py --sobel --source "resized_dataset"

dataset/consts.py

@@ -0,0 +1,40 @@
+PLANT_CLASSES = [
+    "Tomato",
+    "Potato",
+    "Corn_(maize)",
+    "Apple",
+    "Blueberry",
+    "Soybean",
+    "Cherry_(including_sour)",
+    "Squash",
+    "Strawberry",
+    "Pepper,_bell",
+    "Peach",
+    "Grape",
+    "Orange",
+    "Raspberry",
+]
+
+DISEASE_CLASSES = [
+    "healthy",
+    "Northern_Leaf_Blight",
+    "Tomato_mosaic_virus",
+    "Early_blight",
+    "Leaf_scorch",
+    "Tomato_Yellow_Leaf_Curl_Virus",
+    "Cedar_apple_rust",
+    "Late_blight",
+    "Spider_mites Two-spotted_spider_mite",
+    "Black_rot",
+    "Bacterial_spot",
+    "Apple_scab",
+    "Powdery_mildew",
+    "Esca_(Black_Measles)",
+    "Haunglongbing_(Citrus_greening)",
+    "Leaf_Mold",
+    "Common_rust_",
+    "Target_Spot",
+    "Leaf_blight_(Isariopsis_Leaf_Spot)",
+    "Septoria_leaf_spot",
+    "Cercospora_leaf_spot Gray_leaf_spot",
+]

dataset/dataset.py

@@ -0,0 +1,75 @@
+import os
+from pathlib import Path
+
+import tensorflow as tf
+
+from .consts import DISEASE_CLASSES, PLANT_CLASSES
+
+
+class Dataset:
+    ''' Class to load and preprocess the dataset. 
+    Loads images and labels from the given directory to tf.data.Dataset.
+
+
+    Args:
+        `data_dir (Path)`: Path to the dataset directory.
+        `seed (int)`: Seed for shuffling the dataset.
+        `repeat (int)`: Number of times to repeat the dataset.
+        `shuffle_buffer_size (int)`: Size of the buffer for shuffling the dataset.
+        `batch_size (int)`: Batch size for the dataset.
+    '''
+
+    def __init__(self,
+                 data_dir: Path,
+                 seed: int = 42,
+                 repeat: int = 1,
+                 shuffle_buffer_size: int = 10_000,
+                 batch_size: int = 64) -> None:
+        self.data_dir = data_dir
+        self.seed = seed
+        self.repeat = repeat
+        self.shuffle_buffer_size = shuffle_buffer_size
+        self.batch_size = batch_size
+
+        self.dataset = self.__load_dataset()\
+            .shuffle(self.shuffle_buffer_size, seed=self.seed)\
+            .repeat(self.repeat)\
+            .batch(self.batch_size, drop_remainder=True)\
+            .prefetch(tf.data.experimental.AUTOTUNE)
+
+    def __load_dataset(self) -> tf.data.Dataset:
+        # check if path has 'test' word in it
+        dataset = tf.data.Dataset.list_files(str(self.data_dir / '*/*'))
+        if 'test' in str(self.data_dir).lower():
+            # file names issue - labels have camel case (regex?) and differs from the train/valid sets
+            pass
+        else:
+            dataset = dataset.map(
+                self.__preprocess, num_parallel_calls=tf.data.experimental.AUTOTUNE)
+
+        return dataset
+
+    def __get_labels(self, image_path):
+        path = tf.strings.split(image_path, os.path.sep)[-2]
+        plant = tf.strings.split(path, '___')[0]
+        disease = tf.strings.split(path, '___')[1]
+
+        one_hot_plant = plant == PLANT_CLASSES
+        one_hot_disease = disease == DISEASE_CLASSES
+
+        return tf.cast(one_hot_plant, dtype=tf.uint8, name=None), tf.cast(one_hot_disease, dtype=tf.uint8, name=None)
+
+    def __get_image(self, image_path):
+        img = tf.io.read_file(image_path)
+        img = tf.io.decode_jpeg(img, channels=3)
+        return tf.cast(img, dtype=tf.float32, name=None) / 255.
+
+    def __preprocess(self, image_path):
+        labels = self.__get_labels(image_path)
+        image = self.__get_image(image_path)
+
+        # returns X, Y1, Y2
+        return image, labels[0], labels[1]
+
+    def __getattr__(self, attr):
+        return getattr(self.dataset, attr)

file_manager/data_manager.py

@@ -0,0 +1,112 @@
+import argparse
+import glob
+import os
+import shutil
+from pathlib import Path
+from zipfile import ZipFile
+
+import cv2
+import wget
+
+main_path = Path("data/")
+path_to_train_and_valid = main_path / "%s/**/*.*"
+original_dataset_name = "original_dataset"
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--download", action="store_true",
+                    help="Download the data")
+parser.add_argument("--resize", action="store_true",
+                    help="Resize the dataset")
+parser.add_argument("--shape", type=int, nargs="+", default=(64, 64),
+                    help="Shape of the resized images. Applied only for resize option. Default: (64, 64)")
+parser.add_argument("--sobel", action="store_true",
+                    help="Apply Sobel filter to the dataset")
+parser.add_argument("--source", type=str, default="original_dataset",
+                    help="Name of the source dataset. Applied for all arguments except download. Default: original_dataset")
+args = parser.parse_args()
+
+
+class DataManager:
+
+    def download_data(self):
+        if not os.path.isfile("archive.zip"):
+            wget.download("https://storage.googleapis.com/kaggle-data-sets/78313/182633/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20240502%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20240502T181500Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=87d0661313e358206b6e10d44f135d41e23501d601e58b1e8236ca28a82ccc434534564b45baa84c4d829dd1995ff384d51fe5dba3f543d00eb0763169fd712c6c8f91bb4f298db38a19b31b2d489798a9723a271aa4108d7b93345c5a64a7ef00b9b8f27d1d5f728e373c870f0287eb89bc747941f0aeeb4703c288059e2e07b7ece3a83114a9607276874a90d4ec96dde06fddb94a0d3af72848565661b1404e3ea248eeebf46374daada7df1f37db7d62b21b4ac90706ea64cc74200a58f35bfe379703e7691aeda9e39635b02f58a9f8399fa64b031b1a9bccd7f109d256c6f4886ef94fcdc11034d6da13c0f1d4d8b97cabdd295862a5107b587824ebe8")
+
+    def unzip_data(self, file_name, path_to_extract):
+        full_path_to_extract = main_path / path_to_extract
+        old_path = "New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)"
+        if not os.path.exists(main_path):
+            os.makedirs(main_path)
+        ZipFile(file_name).extractall(full_path_to_extract)
+        # shutil.move("data/test/test",
+        #             full_path_to_extract, copy_function=shutil.copytree)
+        shutil.move(full_path_to_extract / old_path / "train",
+                    full_path_to_extract / "train", copy_function=shutil.copytree)
+        shutil.move(full_path_to_extract / old_path / "valid",
+                    full_path_to_extract / "valid", copy_function=shutil.copytree)
+        shutil.rmtree(
+            full_path_to_extract / "New Plant Diseases Dataset(Augmented)"
+        )
+        shutil.rmtree(
+            full_path_to_extract / "new plant diseases dataset(augmented)"
+        )
+        shutil.rmtree(full_path_to_extract / "test")
+        self.get_test_ds_from_validation()
+
+    def write_image(self, image, path):
+        os.makedirs(path.rsplit('/', 1)[0], exist_ok=True)
+        cv2.imwrite(path, image)
+
+    def get_test_ds_from_validation(self, files_per_category: int = 2):
+        path_to_extract = main_path / original_dataset_name
+        valid_ds = glob.glob(str(path_to_extract / "valid/*/*"))
+
+        category_dirs = set([category_dir.split("/")[-2]
+                            for category_dir in valid_ds])
+        category_lists = {category: [] for category in category_dirs}
+        for file_path in valid_ds:
+            category = file_path.split("/")[-2]
+            category_lists[category].append(file_path)
+
+        test_dir = path_to_extract / "test"
+        if not os.path.exists(test_dir):
+            os.makedirs(test_dir, exist_ok=True)
+
+        for category, files in category_lists.items():
+            os.makedirs(test_dir / category, exist_ok=True)
+            files.sort()
+            for file in files[:files_per_category]:
+                shutil.move(file, test_dir / category)
+
+    def resize_dataset(self, source_dataset_name, shape):
+        dataset_name = "resized_dataset"
+        if not os.path.exists(main_path / dataset_name):
+            for file in glob.glob(str(path_to_train_and_valid) % source_dataset_name, recursive=True):
+                path_to_file = file.replace("\\", "/")
+                image = cv2.imread(path_to_file)
+                image = cv2.resize(image, shape)
+                new_path = path_to_file.replace(
+                    source_dataset_name, dataset_name)
+                self.write_image(image, new_path)
+
+    def sobelx(self, source_dataset_name):
+        dataset_name = "sobel_dataset"
+        if not os.path.exists(main_path / dataset_name):
+            for file in glob.glob(str(path_to_train_and_valid) % source_dataset_name, recursive=True):
+                path_to_file = file.replace("\\", "/")
+                image = cv2.imread(path_to_file)
+                sobel = cv2.Sobel(image, cv2.CV_64F, 1, 0, ksize=5)
+                new_path = path_to_file.replace(
+                    source_dataset_name, dataset_name)
+                self.write_image(sobel, new_path)
+
+
+if __name__ == "__main__":
+    data_manager = DataManager()
+    if args.download:
+        data_manager.download_data()
+        data_manager.unzip_data("archive.zip", original_dataset_name)
+    if args.resize:
+        data_manager.resize_dataset(args.source, tuple(args.shape))
+    if args.sobel:
+        data_manager.sobelx(args.source)

file_manager/shard_files.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+
+# TODO: split the files into smaller dirs and make list of them
+class FileSharder:
+    def __init__(self,
+                 train_dir: Path = Path('./data/resized_dataset/train'),
+                 valid_dir: Path = Path('./data/resized_dataset/valid'),
+                 test_dir: Path = Path('./data/resized_dataset/test'),
+                 shard_size = 5_000) -> None:
+        self.shard_size = shard_size
+
+        self.train_dir = train_dir
+        self.valid_dir = valid_dir
+        self.test_dir = test_dir
+
+        self.shard()
+
+    def shard(self):
+        pass

requirements.txt

@@ -0,0 +1,4 @@
+tensorflow==2.16.1
+numpy==1.26.4
+opencv-python==4.9.0.80
+wget==3.2

test.py

@@ -0,0 +1,10 @@
+
+from pathlib import Path
+
+from dataset.dataset import Dataset
+
+train_dataset = Dataset(Path('data/resized_dataset/train'))
+valid_dataset = Dataset(Path('data/resized_dataset/valid'))
+
+for i in train_dataset.take(1):
+    print(i)