diff --git a/.gitignore b/.gitignore index 6f951ce..ac91879 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,54 @@ -data -archive.zip \ No newline at end of file +data/ +*.zip + +# https://github.com/microsoft/vscode-python/blob/main/.gitignore +.DS_Store +.huskyrc.json +out +log.log +**/node_modules +*.pyc +*.vsix +envVars.txt +**/.vscode/.ropeproject/** +**/testFiles/**/.cache/** +*.noseids +.nyc_output +.vscode-test +__pycache__ +npm-debug.log +**/.mypy_cache/** +!yarn.lock +coverage/ +cucumber-report.json +**/.vscode-test/** +**/.vscode test/** +**/.vscode-smoke/** +**/.venv*/ +port.txt +precommit.hook +python_files/lib/** +python_files/get-pip.py +debug_coverage*/** +languageServer/** +languageServer.*/** +bin/** +obj/** +.pytest_cache +tmp/** +.python-version +.vs/ +test-results*.xml +xunit-test-results.xml +build/ci/performance/performance-results.json +!build/ +debug*.log +debugpy*.log +pydevd*.log +nodeLanguageServer/** +nodeLanguageServer.*/** +dist/** +# translation files +*.xlf +package.nls.*.json +l10n/ \ No newline at end of file diff --git a/DataManager.py b/DataManager.py deleted file mode 100644 index d837ec8..0000000 --- a/DataManager.py +++ /dev/null @@ -1,55 +0,0 @@ -import glob -import shutil -import cv2 -from zipfile import ZipFile -import os -import wget - -mainPath="data/" -pathToTrainAndValidDate = mainPath + "%s/**/*.*" -pathToTestDataset = mainPath + "/test" -originalDatasetName = "original dataset" - -class DataManager: - - def downloadData(self): - if not os.path.isfile("archive.zip"): - wget.download("https://storage.googleapis.com/kaggle-data-sets/78313/182633/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20240502%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20240502T181500Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=87d0661313e358206b6e10d44f135d41e23501d601e58b1e8236ca28a82ccc434534564b45baa84c4d829dd1995ff384d51fe5dba3f543d00eb0763169fd712c6c8f91bb4f298db38a19b31b2d489798a9723a271aa4108d7b93345c5a64a7ef00b9b8f27d1d5f728e373c870f0287eb89bc747941f0aeeb4703c288059e2e07b7ece3a83114a9607276874a90d4ec96dde06fddb94a0d3af72848565661b1404e3ea248eeebf46374daada7df1f37db7d62b21b4ac90706ea64cc74200a58f35bfe379703e7691aeda9e39635b02f58a9f8399fa64b031b1a9bccd7f109d256c6f4886ef94fcdc11034d6da13c0f1d4d8b97cabdd295862a5107b587824ebe8") - - def unzipData(self, fileName, pathToExtract): - if not os.path.exists(mainPath): - os.makedirs("data") - ZipFile(fileName).extractall(mainPath + pathToExtract) - shutil.move("data/original dataset/test/test", "data", copy_function = shutil.copytree) - shutil.move("data/original dataset/New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)/train", "data/original dataset/train", copy_function = shutil.copytree) - shutil.move("data/original dataset/New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)/valid", "data/original dataset/valid", copy_function = shutil.copytree) - shutil.rmtree("data/original dataset/New Plant Diseases Dataset(Augmented)") - shutil.rmtree("data/Detection-of-plant-diseases/data/original dataset/test") - - def writeImageToGivenPath(self, image, path): - os.makedirs(path.rsplit('/', 1)[0], exist_ok=True) - cv2.imwrite(path, image) - - def resizeDataset(self, soruceDatasetName, width, height): - if not os.path.exists(mainPath + "resized dataset"): - for file in glob.glob(pathToTrainAndValidDate % soruceDatasetName, recursive=True): - pathToFile = file.replace("\\","/") - image = cv2.imread(pathToFile) - image = cv2.resize(image, (width, height)) - newPath = pathToFile.replace(soruceDatasetName,"resized dataset") - self.writeImageToGivenPath(image,newPath) - - def sobelx(self, soruceDatasetName): - if not os.path.exists(mainPath + "sobel dataset"): - for file in glob.glob(pathToTrainAndValidDate % soruceDatasetName, recursive=True): - pathToFile = file.replace("\\","/") - image = cv2.imread(pathToFile) - sobel = cv2.Sobel(image,cv2.CV_64F,1,0,ksize=5) - newPath = pathToFile.replace(soruceDatasetName,"sobel dataset") - self.writeImageToGivenPath(sobel,newPath) - -dataManager = DataManager() -dataManager.downloadData() -dataManager.unzipData("archive.zip","original dataset") -dataManager.resizeDataset("original dataset", 64, 64) -dataManager.sobelx("resized dataset") \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5734438 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +.PHONY: download-dataset sobel-dataset + +download-dataset: + python3 ./file_manager/data_manager.py --download + +sobel-dataset: + python3 ./file_manager/data_manager.py --sobel \ No newline at end of file diff --git a/file_manager/__init__.py b/file_manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/file_manager/data_manager.py b/file_manager/data_manager.py new file mode 100644 index 0000000..9136161 --- /dev/null +++ b/file_manager/data_manager.py @@ -0,0 +1,83 @@ +import glob +import shutil +import cv2 +from zipfile import ZipFile +import os +import wget +import argparse +from pathlib import Path + +main_path = Path("data/") +path_to_train_and_valid = main_path / "%s/**/*.*" +path_to_test_dataset = main_path / "test" +original_dataset_name = "original_dataset" + +parser = argparse.ArgumentParser() +parser.add_argument("--download", action="store_true", + help="Download the data") +parser.add_argument("--sobel", action="store_true", + help="Apply Sobel filter to the dataset") + +args = parser.parse_args() + + +class DataManager: + + def download_data(self): + if not os.path.isfile("archive.zip"): + wget.download("https://storage.googleapis.com/kaggle-data-sets/78313/182633/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20240502%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20240502T181500Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=87d0661313e358206b6e10d44f135d41e23501d601e58b1e8236ca28a82ccc434534564b45baa84c4d829dd1995ff384d51fe5dba3f543d00eb0763169fd712c6c8f91bb4f298db38a19b31b2d489798a9723a271aa4108d7b93345c5a64a7ef00b9b8f27d1d5f728e373c870f0287eb89bc747941f0aeeb4703c288059e2e07b7ece3a83114a9607276874a90d4ec96dde06fddb94a0d3af72848565661b1404e3ea248eeebf46374daada7df1f37db7d62b21b4ac90706ea64cc74200a58f35bfe379703e7691aeda9e39635b02f58a9f8399fa64b031b1a9bccd7f109d256c6f4886ef94fcdc11034d6da13c0f1d4d8b97cabdd295862a5107b587824ebe8") + + def unzip_data(self, file_name, path_to_extract): + full_path_to_extract = main_path / path_to_extract + old_path = "New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)" + if not os.path.exists(main_path): + os.makedirs(main_path) + ZipFile(file_name).extractall(full_path_to_extract) + # shutil.move("data/test/test", + # full_path_to_extract, copy_function=shutil.copytree) + shutil.move(full_path_to_extract / old_path / "train", + full_path_to_extract / "train", copy_function=shutil.copytree) + shutil.move(full_path_to_extract / old_path / "valid", + full_path_to_extract / "valid", copy_function=shutil.copytree) + shutil.rmtree( + full_path_to_extract / "New Plant Diseases Dataset(Augmented)" + ) + shutil.rmtree( + full_path_to_extract / "new plant diseases dataset(augmented)" + ) + + def write_image(self, image, path): + os.makedirs(path.rsplit('/', 1)[0], exist_ok=True) + cv2.imwrite(path, image) + + def resize_dataset(self, source_dataset_name, width, height): + dataset_name = "resized_dataset" + if not os.path.exists(main_path / dataset_name): + for file in glob.glob(str(path_to_train_and_valid) % source_dataset_name, recursive=True): + path_to_file = file.replace("\\", "/") + image = cv2.imread(path_to_file) + image = cv2.resize(image, (width, height)) + new_path = path_to_file.replace( + source_dataset_name, dataset_name) + self.write_image(image, new_path) + + def sobelx(self, source_dataset_name): + dataset_name = "sobel_dataset" + if not os.path.exists(main_path / dataset_name): + for file in glob.glob(str(path_to_train_and_valid) % source_dataset_name, recursive=True): + path_to_file = file.replace("\\", "/") + image = cv2.imread(path_to_file) + sobel = cv2.Sobel(image, cv2.CV_64F, 1, 0, ksize=5) + new_path = path_to_file.replace( + source_dataset_name, dataset_name) + self.write_image(sobel, new_path) + + +if __name__ == "__main__": + data_manager = DataManager() + if args.download: + data_manager.download_data() + data_manager.unzip_data("archive.zip", original_dataset_name) + data_manager.resize_dataset(original_dataset_name, 64, 64) + if args.sobel: + data_manager.sobelx("resized_dataset") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cd27218 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +tensorflow==2.16.1 +tensorflow-io==0.37.0 +numpy==1.26.4 +opencv-python==4.9.0.80 +wget==3.2