Merge pull request 'feature/data_manager' (#1) from feature/data_manager into main
Reviewed-on: #1 Reviewed-by: s495727 <krzboj@st.amu.edu.pl>
This commit is contained in:
commit
040b1d014f
56
.gitignore
vendored
56
.gitignore
vendored
@ -1,2 +1,54 @@
|
|||||||
data
|
data/
|
||||||
archive.zip
|
*.zip
|
||||||
|
|
||||||
|
# https://github.com/microsoft/vscode-python/blob/main/.gitignore
|
||||||
|
.DS_Store
|
||||||
|
.huskyrc.json
|
||||||
|
out
|
||||||
|
log.log
|
||||||
|
**/node_modules
|
||||||
|
*.pyc
|
||||||
|
*.vsix
|
||||||
|
envVars.txt
|
||||||
|
**/.vscode/.ropeproject/**
|
||||||
|
**/testFiles/**/.cache/**
|
||||||
|
*.noseids
|
||||||
|
.nyc_output
|
||||||
|
.vscode-test
|
||||||
|
__pycache__
|
||||||
|
npm-debug.log
|
||||||
|
**/.mypy_cache/**
|
||||||
|
!yarn.lock
|
||||||
|
coverage/
|
||||||
|
cucumber-report.json
|
||||||
|
**/.vscode-test/**
|
||||||
|
**/.vscode test/**
|
||||||
|
**/.vscode-smoke/**
|
||||||
|
**/.venv*/
|
||||||
|
port.txt
|
||||||
|
precommit.hook
|
||||||
|
python_files/lib/**
|
||||||
|
python_files/get-pip.py
|
||||||
|
debug_coverage*/**
|
||||||
|
languageServer/**
|
||||||
|
languageServer.*/**
|
||||||
|
bin/**
|
||||||
|
obj/**
|
||||||
|
.pytest_cache
|
||||||
|
tmp/**
|
||||||
|
.python-version
|
||||||
|
.vs/
|
||||||
|
test-results*.xml
|
||||||
|
xunit-test-results.xml
|
||||||
|
build/ci/performance/performance-results.json
|
||||||
|
!build/
|
||||||
|
debug*.log
|
||||||
|
debugpy*.log
|
||||||
|
pydevd*.log
|
||||||
|
nodeLanguageServer/**
|
||||||
|
nodeLanguageServer.*/**
|
||||||
|
dist/**
|
||||||
|
# translation files
|
||||||
|
*.xlf
|
||||||
|
package.nls.*.json
|
||||||
|
l10n/
|
||||||
@ -1,55 +0,0 @@
|
|||||||
import glob
|
|
||||||
import shutil
|
|
||||||
import cv2
|
|
||||||
from zipfile import ZipFile
|
|
||||||
import os
|
|
||||||
import wget
|
|
||||||
|
|
||||||
mainPath="data/"
|
|
||||||
pathToTrainAndValidDate = mainPath + "%s/**/*.*"
|
|
||||||
pathToTestDataset = mainPath + "/test"
|
|
||||||
originalDatasetName = "original dataset"
|
|
||||||
|
|
||||||
class DataManager:
|
|
||||||
|
|
||||||
def downloadData(self):
|
|
||||||
if not os.path.isfile("archive.zip"):
|
|
||||||
wget.download("https://storage.googleapis.com/kaggle-data-sets/78313/182633/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20240502%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20240502T181500Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=87d0661313e358206b6e10d44f135d41e23501d601e58b1e8236ca28a82ccc434534564b45baa84c4d829dd1995ff384d51fe5dba3f543d00eb0763169fd712c6c8f91bb4f298db38a19b31b2d489798a9723a271aa4108d7b93345c5a64a7ef00b9b8f27d1d5f728e373c870f0287eb89bc747941f0aeeb4703c288059e2e07b7ece3a83114a9607276874a90d4ec96dde06fddb94a0d3af72848565661b1404e3ea248eeebf46374daada7df1f37db7d62b21b4ac90706ea64cc74200a58f35bfe379703e7691aeda9e39635b02f58a9f8399fa64b031b1a9bccd7f109d256c6f4886ef94fcdc11034d6da13c0f1d4d8b97cabdd295862a5107b587824ebe8")
|
|
||||||
|
|
||||||
def unzipData(self, fileName, pathToExtract):
|
|
||||||
if not os.path.exists(mainPath):
|
|
||||||
os.makedirs("data")
|
|
||||||
ZipFile(fileName).extractall(mainPath + pathToExtract)
|
|
||||||
shutil.move("data/original dataset/test/test", "data", copy_function = shutil.copytree)
|
|
||||||
shutil.move("data/original dataset/New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)/train", "data/original dataset/train", copy_function = shutil.copytree)
|
|
||||||
shutil.move("data/original dataset/New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)/valid", "data/original dataset/valid", copy_function = shutil.copytree)
|
|
||||||
shutil.rmtree("data/original dataset/New Plant Diseases Dataset(Augmented)")
|
|
||||||
shutil.rmtree("data/Detection-of-plant-diseases/data/original dataset/test")
|
|
||||||
|
|
||||||
def writeImageToGivenPath(self, image, path):
|
|
||||||
os.makedirs(path.rsplit('/', 1)[0], exist_ok=True)
|
|
||||||
cv2.imwrite(path, image)
|
|
||||||
|
|
||||||
def resizeDataset(self, soruceDatasetName, width, height):
|
|
||||||
if not os.path.exists(mainPath + "resized dataset"):
|
|
||||||
for file in glob.glob(pathToTrainAndValidDate % soruceDatasetName, recursive=True):
|
|
||||||
pathToFile = file.replace("\\","/")
|
|
||||||
image = cv2.imread(pathToFile)
|
|
||||||
image = cv2.resize(image, (width, height))
|
|
||||||
newPath = pathToFile.replace(soruceDatasetName,"resized dataset")
|
|
||||||
self.writeImageToGivenPath(image,newPath)
|
|
||||||
|
|
||||||
def sobelx(self, soruceDatasetName):
|
|
||||||
if not os.path.exists(mainPath + "sobel dataset"):
|
|
||||||
for file in glob.glob(pathToTrainAndValidDate % soruceDatasetName, recursive=True):
|
|
||||||
pathToFile = file.replace("\\","/")
|
|
||||||
image = cv2.imread(pathToFile)
|
|
||||||
sobel = cv2.Sobel(image,cv2.CV_64F,1,0,ksize=5)
|
|
||||||
newPath = pathToFile.replace(soruceDatasetName,"sobel dataset")
|
|
||||||
self.writeImageToGivenPath(sobel,newPath)
|
|
||||||
|
|
||||||
dataManager = DataManager()
|
|
||||||
dataManager.downloadData()
|
|
||||||
dataManager.unzipData("archive.zip","original dataset")
|
|
||||||
dataManager.resizeDataset("original dataset", 64, 64)
|
|
||||||
dataManager.sobelx("resized dataset")
|
|
||||||
7
Makefile
Normal file
7
Makefile
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
.PHONY: download-dataset sobel-dataset
|
||||||
|
|
||||||
|
download-dataset:
|
||||||
|
python3 ./file_manager/data_manager.py --download
|
||||||
|
|
||||||
|
sobel-dataset:
|
||||||
|
python3 ./file_manager/data_manager.py --sobel
|
||||||
0
file_manager/__init__.py
Normal file
0
file_manager/__init__.py
Normal file
83
file_manager/data_manager.py
Normal file
83
file_manager/data_manager.py
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
import glob
|
||||||
|
import shutil
|
||||||
|
import cv2
|
||||||
|
from zipfile import ZipFile
|
||||||
|
import os
|
||||||
|
import wget
|
||||||
|
import argparse
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
main_path = Path("data/")
|
||||||
|
path_to_train_and_valid = main_path / "%s/**/*.*"
|
||||||
|
path_to_test_dataset = main_path / "test"
|
||||||
|
original_dataset_name = "original_dataset"
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--download", action="store_true",
|
||||||
|
help="Download the data")
|
||||||
|
parser.add_argument("--sobel", action="store_true",
|
||||||
|
help="Apply Sobel filter to the dataset")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
class DataManager:
|
||||||
|
|
||||||
|
def download_data(self):
|
||||||
|
if not os.path.isfile("archive.zip"):
|
||||||
|
wget.download("https://storage.googleapis.com/kaggle-data-sets/78313/182633/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20240502%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20240502T181500Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=87d0661313e358206b6e10d44f135d41e23501d601e58b1e8236ca28a82ccc434534564b45baa84c4d829dd1995ff384d51fe5dba3f543d00eb0763169fd712c6c8f91bb4f298db38a19b31b2d489798a9723a271aa4108d7b93345c5a64a7ef00b9b8f27d1d5f728e373c870f0287eb89bc747941f0aeeb4703c288059e2e07b7ece3a83114a9607276874a90d4ec96dde06fddb94a0d3af72848565661b1404e3ea248eeebf46374daada7df1f37db7d62b21b4ac90706ea64cc74200a58f35bfe379703e7691aeda9e39635b02f58a9f8399fa64b031b1a9bccd7f109d256c6f4886ef94fcdc11034d6da13c0f1d4d8b97cabdd295862a5107b587824ebe8")
|
||||||
|
|
||||||
|
def unzip_data(self, file_name, path_to_extract):
|
||||||
|
full_path_to_extract = main_path / path_to_extract
|
||||||
|
old_path = "New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)"
|
||||||
|
if not os.path.exists(main_path):
|
||||||
|
os.makedirs(main_path)
|
||||||
|
ZipFile(file_name).extractall(full_path_to_extract)
|
||||||
|
# shutil.move("data/test/test",
|
||||||
|
# full_path_to_extract, copy_function=shutil.copytree)
|
||||||
|
shutil.move(full_path_to_extract / old_path / "train",
|
||||||
|
full_path_to_extract / "train", copy_function=shutil.copytree)
|
||||||
|
shutil.move(full_path_to_extract / old_path / "valid",
|
||||||
|
full_path_to_extract / "valid", copy_function=shutil.copytree)
|
||||||
|
shutil.rmtree(
|
||||||
|
full_path_to_extract / "New Plant Diseases Dataset(Augmented)"
|
||||||
|
)
|
||||||
|
shutil.rmtree(
|
||||||
|
full_path_to_extract / "new plant diseases dataset(augmented)"
|
||||||
|
)
|
||||||
|
|
||||||
|
def write_image(self, image, path):
|
||||||
|
os.makedirs(path.rsplit('/', 1)[0], exist_ok=True)
|
||||||
|
cv2.imwrite(path, image)
|
||||||
|
|
||||||
|
def resize_dataset(self, source_dataset_name, width, height):
|
||||||
|
dataset_name = "resized_dataset"
|
||||||
|
if not os.path.exists(main_path / dataset_name):
|
||||||
|
for file in glob.glob(str(path_to_train_and_valid) % source_dataset_name, recursive=True):
|
||||||
|
path_to_file = file.replace("\\", "/")
|
||||||
|
image = cv2.imread(path_to_file)
|
||||||
|
image = cv2.resize(image, (width, height))
|
||||||
|
new_path = path_to_file.replace(
|
||||||
|
source_dataset_name, dataset_name)
|
||||||
|
self.write_image(image, new_path)
|
||||||
|
|
||||||
|
def sobelx(self, source_dataset_name):
|
||||||
|
dataset_name = "sobel_dataset"
|
||||||
|
if not os.path.exists(main_path / dataset_name):
|
||||||
|
for file in glob.glob(str(path_to_train_and_valid) % source_dataset_name, recursive=True):
|
||||||
|
path_to_file = file.replace("\\", "/")
|
||||||
|
image = cv2.imread(path_to_file)
|
||||||
|
sobel = cv2.Sobel(image, cv2.CV_64F, 1, 0, ksize=5)
|
||||||
|
new_path = path_to_file.replace(
|
||||||
|
source_dataset_name, dataset_name)
|
||||||
|
self.write_image(sobel, new_path)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
data_manager = DataManager()
|
||||||
|
if args.download:
|
||||||
|
data_manager.download_data()
|
||||||
|
data_manager.unzip_data("archive.zip", original_dataset_name)
|
||||||
|
data_manager.resize_dataset(original_dataset_name, 64, 64)
|
||||||
|
if args.sobel:
|
||||||
|
data_manager.sobelx("resized_dataset")
|
||||||
5
requirements.txt
Normal file
5
requirements.txt
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
tensorflow==2.16.1
|
||||||
|
tensorflow-io==0.37.0
|
||||||
|
numpy==1.26.4
|
||||||
|
opencv-python==4.9.0.80
|
||||||
|
wget==3.2
|
||||||
Loading…
Reference in New Issue
Block a user