s495728
/
Detection-of-plant-diseases
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' of git.wmi.amu.edu.pl:s495728/Detection-of-plant-diseases into feature/basic-model-setup

This commit is contained in:
mszmyd 2024-05-06 00:10:40 +02:00
parent 09070879ac 7849d9ad51
commit e9fffa0539
11 changed files with 324 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
.gitignore vendored
View File

@ -1,4 +1,55 @@
data
archive.zip
.ipynb_checkpoints
__pycache__
data/
*.zip
# https://github.com/microsoft/vscode-python/blob/main/.gitignore
.DS_Store
.huskyrc.json
out
log.log
**/node_modules
*.pyc
*.vsix
envVars.txt
**/.vscode/.ropeproject/**
**/testFiles/**/.cache/**
*.noseids
.nyc_output
.vscode-test
__pycache__
npm-debug.log
**/.mypy_cache/**
!yarn.lock
coverage/
cucumber-report.json
**/.vscode-test/**
**/.vscode test/**
**/.vscode-smoke/**
**/.venv*/
port.txt
precommit.hook
python_files/lib/**
python_files/get-pip.py
debug_coverage*/**
languageServer/**
languageServer.*/**
bin/**
obj/**
.pytest_cache
tmp/**
.python-version
.vs/
test-results*.xml
xunit-test-results.xml
build/ci/performance/performance-results.json
!build/
debug*.log
debugpy*.log
pydevd*.log
nodeLanguageServer/**
nodeLanguageServer.*/**
dist/**
# translation files
*.xlf
package.nls.*.json
l10n/

55
DataManager.py
View File

@ -1,55 +0,0 @@
import glob
import shutil
import cv2
from zipfile import ZipFile
import os
import wget
mainPath="data/"
pathToTrainAndValidDate = mainPath + "%s/**/*.*"
pathToTestDataset = mainPath + "/test"
originalDatasetName = "original dataset"
class DataManager:
def downloadData(self):
if not os.path.isfile("archive.zip"):
wget.download("https://storage.googleapis.com/kaggle-data-sets/78313/182633/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20240502%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20240502T181500Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=87d0661313e358206b6e10d44f135d41e23501d601e58b1e8236ca28a82ccc434534564b45baa84c4d829dd1995ff384d51fe5dba3f543d00eb0763169fd712c6c8f91bb4f298db38a19b31b2d489798a9723a271aa4108d7b93345c5a64a7ef00b9b8f27d1d5f728e373c870f0287eb89bc747941f0aeeb4703c288059e2e07b7ece3a83114a9607276874a90d4ec96dde06fddb94a0d3af72848565661b1404e3ea248eeebf46374daada7df1f37db7d62b21b4ac90706ea64cc74200a58f35bfe379703e7691aeda9e39635b02f58a9f8399fa64b031b1a9bccd7f109d256c6f4886ef94fcdc11034d6da13c0f1d4d8b97cabdd295862a5107b587824ebe8")
def unzipData(self, fileName, pathToExtract):
if not os.path.exists(mainPath):
os.makedirs("data")
ZipFile(fileName).extractall(mainPath + pathToExtract)
shutil.move("data/original dataset/test/test", "data", copy_function = shutil.copytree)
shutil.move("data/original dataset/New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)/train", "data/original dataset/train", copy_function = shutil.copytree)
shutil.move("data/original dataset/New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)/valid", "data/original dataset/valid", copy_function = shutil.copytree)
shutil.rmtree("data/original dataset/New Plant Diseases Dataset(Augmented)")
shutil.rmtree("data/Detection-of-plant-diseases/data/original dataset/test")
def writeImageToGivenPath(self, image, path):
os.makedirs(path.rsplit('/', 1)[0], exist_ok=True)
cv2.imwrite(path, image)
def resizeDataset(self, soruceDatasetName, width, height):
if not os.path.exists(mainPath + "resized dataset"):
for file in glob.glob(pathToTrainAndValidDate % soruceDatasetName, recursive=True):
pathToFile = file.replace("\\","/")
image = cv2.imread(pathToFile)
image = cv2.resize(image, (width, height))
newPath = pathToFile.replace(soruceDatasetName,"resized dataset")
self.writeImageToGivenPath(image,newPath)
def sobelx(self, soruceDatasetName):
if not os.path.exists(mainPath + "sobel dataset"):
for file in glob.glob(pathToTrainAndValidDate % soruceDatasetName, recursive=True):
pathToFile = file.replace("\\","/")
image = cv2.imread(pathToFile)
sobel = cv2.Sobel(image,cv2.CV_64F,1,0,ksize=5)
newPath = pathToFile.replace(soruceDatasetName,"sobel dataset")
self.writeImageToGivenPath(sobel,newPath)
dataManager = DataManager()
dataManager.downloadData()
dataManager.unzipData("archive.zip","original dataset")
dataManager.resizeDataset("original dataset", 64, 64)
dataManager.sobelx("resized dataset")

10
Makefile Normal file
View File

@ -0,0 +1,10 @@
.PHONY: download-dataset resize-dataset sobel-dataset
download-dataset:
python3 ./file_manager/data_manager.py --download
resize-dataset:
python3 ./file_manager/data_manager.py --resize --shape 64 64 --source "original_dataset"
sobel-dataset:
python3 ./file_manager/data_manager.py --sobel --source "resized_dataset"

0
dataset/__init__.py Normal file
View File

40
dataset/consts.py Normal file
View File

@ -0,0 +1,40 @@
PLANT_CLASSES = [
"Tomato",
"Potato",
"Corn_(maize)",
"Apple",
"Blueberry",
"Soybean",
"Cherry_(including_sour)",
"Squash",
"Strawberry",
"Pepper,_bell",
"Peach",
"Grape",
"Orange",
"Raspberry",
]
DISEASE_CLASSES = [
"healthy",
"Northern_Leaf_Blight",
"Tomato_mosaic_virus",
"Early_blight",
"Leaf_scorch",
"Tomato_Yellow_Leaf_Curl_Virus",
"Cedar_apple_rust",
"Late_blight",
"Spider_mites Two-spotted_spider_mite",
"Black_rot",
"Bacterial_spot",
"Apple_scab",
"Powdery_mildew",
"Esca_(Black_Measles)",
"Haunglongbing_(Citrus_greening)",
"Leaf_Mold",
"Common_rust_",
"Target_Spot",
"Leaf_blight_(Isariopsis_Leaf_Spot)",
"Septoria_leaf_spot",
"Cercospora_leaf_spot Gray_leaf_spot",
]

75
dataset/dataset.py Normal file
View File

@ -0,0 +1,75 @@
import os
from pathlib import Path
import tensorflow as tf
from .consts import DISEASE_CLASSES, PLANT_CLASSES
class Dataset:
''' Class to load and preprocess the dataset.
Loads images and labels from the given directory to tf.data.Dataset.
Args:
`data_dir (Path)`: Path to the dataset directory.
`seed (int)`: Seed for shuffling the dataset.
`repeat (int)`: Number of times to repeat the dataset.
`shuffle_buffer_size (int)`: Size of the buffer for shuffling the dataset.
`batch_size (int)`: Batch size for the dataset.
'''
def __init__(self,
data_dir: Path,
seed: int = 42,
repeat: int = 1,
shuffle_buffer_size: int = 10_000,
batch_size: int = 64) -> None:
self.data_dir = data_dir
self.seed = seed
self.repeat = repeat
self.shuffle_buffer_size = shuffle_buffer_size
self.batch_size = batch_size
self.dataset = self.__load_dataset()\
.shuffle(self.shuffle_buffer_size, seed=self.seed)\
.repeat(self.repeat)\
.batch(self.batch_size, drop_remainder=True)\
.prefetch(tf.data.experimental.AUTOTUNE)
def __load_dataset(self) -> tf.data.Dataset:
# check if path has 'test' word in it
dataset = tf.data.Dataset.list_files(str(self.data_dir / '*/*'))
if 'test' in str(self.data_dir).lower():
# file names issue - labels have camel case (regex?) and differs from the train/valid sets
pass
else:
dataset = dataset.map(
self.__preprocess, num_parallel_calls=tf.data.experimental.AUTOTUNE)
return dataset
def __get_labels(self, image_path):
path = tf.strings.split(image_path, os.path.sep)[-2]
plant = tf.strings.split(path, '___')[0]
disease = tf.strings.split(path, '___')[1]
one_hot_plant = plant == PLANT_CLASSES
one_hot_disease = disease == DISEASE_CLASSES
return tf.cast(one_hot_plant, dtype=tf.uint8, name=None), tf.cast(one_hot_disease, dtype=tf.uint8, name=None)
def __get_image(self, image_path):
img = tf.io.read_file(image_path)
img = tf.io.decode_jpeg(img, channels=3)
return tf.cast(img, dtype=tf.float32, name=None) / 255.
def __preprocess(self, image_path):
labels = self.__get_labels(image_path)
image = self.__get_image(image_path)
# returns X, Y1, Y2
return image, labels[0], labels[1]
def __getattr__(self, attr):
return getattr(self.dataset, attr)

0
file_manager/__init__.py Normal file
View File

112
file_manager/data_manager.py Normal file
View File

@ -0,0 +1,112 @@
import argparse
import glob
import os
import shutil
from pathlib import Path
from zipfile import ZipFile
import cv2
import wget
main_path = Path("data/")
path_to_train_and_valid = main_path / "%s/**/*.*"
original_dataset_name = "original_dataset"
parser = argparse.ArgumentParser()
parser.add_argument("--download", action="store_true",
help="Download the data")
parser.add_argument("--resize", action="store_true",
help="Resize the dataset")
parser.add_argument("--shape", type=int, nargs="+", default=(64, 64),
help="Shape of the resized images. Applied only for resize option. Default: (64, 64)")
parser.add_argument("--sobel", action="store_true",
help="Apply Sobel filter to the dataset")
parser.add_argument("--source", type=str, default="original_dataset",
help="Name of the source dataset. Applied for all arguments except download. Default: original_dataset")
args = parser.parse_args()
class DataManager:
def download_data(self):
if not os.path.isfile("archive.zip"):
wget.download("https://storage.googleapis.com/kaggle-data-sets/78313/182633/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20240502%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20240502T181500Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=87d0661313e358206b6e10d44f135d41e23501d601e58b1e8236ca28a82ccc434534564b45baa84c4d829dd1995ff384d51fe5dba3f543d00eb0763169fd712c6c8f91bb4f298db38a19b31b2d489798a9723a271aa4108d7b93345c5a64a7ef00b9b8f27d1d5f728e373c870f0287eb89bc747941f0aeeb4703c288059e2e07b7ece3a83114a9607276874a90d4ec96dde06fddb94a0d3af72848565661b1404e3ea248eeebf46374daada7df1f37db7d62b21b4ac90706ea64cc74200a58f35bfe379703e7691aeda9e39635b02f58a9f8399fa64b031b1a9bccd7f109d256c6f4886ef94fcdc11034d6da13c0f1d4d8b97cabdd295862a5107b587824ebe8")
def unzip_data(self, file_name, path_to_extract):
full_path_to_extract = main_path / path_to_extract
old_path = "New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)"
if not os.path.exists(main_path):
os.makedirs(main_path)
ZipFile(file_name).extractall(full_path_to_extract)
# shutil.move("data/test/test",
# full_path_to_extract, copy_function=shutil.copytree)
shutil.move(full_path_to_extract / old_path / "train",
full_path_to_extract / "train", copy_function=shutil.copytree)
shutil.move(full_path_to_extract / old_path / "valid",
full_path_to_extract / "valid", copy_function=shutil.copytree)
shutil.rmtree(
full_path_to_extract / "New Plant Diseases Dataset(Augmented)"
)
shutil.rmtree(
full_path_to_extract / "new plant diseases dataset(augmented)"
)
shutil.rmtree(full_path_to_extract / "test")
self.get_test_ds_from_validation()
def write_image(self, image, path):
os.makedirs(path.rsplit('/', 1)[0], exist_ok=True)
cv2.imwrite(path, image)
def get_test_ds_from_validation(self, files_per_category: int = 2):
path_to_extract = main_path / original_dataset_name
valid_ds = glob.glob(str(path_to_extract / "valid/*/*"))
category_dirs = set([category_dir.split("/")[-2]
for category_dir in valid_ds])
category_lists = {category: [] for category in category_dirs}
for file_path in valid_ds:
category = file_path.split("/")[-2]
category_lists[category].append(file_path)
test_dir = path_to_extract / "test"
if not os.path.exists(test_dir):
os.makedirs(test_dir, exist_ok=True)
for category, files in category_lists.items():
os.makedirs(test_dir / category, exist_ok=True)
files.sort()
for file in files[:files_per_category]:
shutil.move(file, test_dir / category)
def resize_dataset(self, source_dataset_name, shape):
dataset_name = "resized_dataset"
if not os.path.exists(main_path / dataset_name):
for file in glob.glob(str(path_to_train_and_valid) % source_dataset_name, recursive=True):
path_to_file = file.replace("\\", "/")
image = cv2.imread(path_to_file)
image = cv2.resize(image, shape)
new_path = path_to_file.replace(
source_dataset_name, dataset_name)
self.write_image(image, new_path)
def sobelx(self, source_dataset_name):
dataset_name = "sobel_dataset"
if not os.path.exists(main_path / dataset_name):
for file in glob.glob(str(path_to_train_and_valid) % source_dataset_name, recursive=True):
path_to_file = file.replace("\\", "/")
image = cv2.imread(path_to_file)
sobel = cv2.Sobel(image, cv2.CV_64F, 1, 0, ksize=5)
new_path = path_to_file.replace(
source_dataset_name, dataset_name)
self.write_image(sobel, new_path)
if __name__ == "__main__":
data_manager = DataManager()
if args.download:
data_manager.download_data()
data_manager.unzip_data("archive.zip", original_dataset_name)
if args.resize:
data_manager.resize_dataset(args.source, tuple(args.shape))
if args.sobel:
data_manager.sobelx(args.source)

19
file_manager/shard_files.py Normal file
View File

@ -0,0 +1,19 @@
from pathlib import Path
# TODO: split the files into smaller dirs and make list of them
class FileSharder:
def __init__(self,
train_dir: Path = Path('./data/resized_dataset/train'),
valid_dir: Path = Path('./data/resized_dataset/valid'),
test_dir: Path = Path('./data/resized_dataset/test'),
shard_size = 5_000) -> None:
self.shard_size = shard_size
self.train_dir = train_dir
self.valid_dir = valid_dir
self.test_dir = test_dir
self.shard()
def shard(self):
pass

4
requirements.txt Normal file
View File

@ -0,0 +1,4 @@
tensorflow==2.16.1
numpy==1.26.4
opencv-python==4.9.0.80
wget==3.2

10
test.py Normal file
View File

@ -0,0 +1,10 @@
from pathlib import Path
from dataset.dataset import Dataset
train_dataset = Dataset(Path('data/resized_dataset/train'))
valid_dataset = Dataset(Path('data/resized_dataset/valid'))
for i in train_dataset.take(1):
print(i)