From 4bb1051bebc8a076e310ad7b1e7e14fb3ddebb72 Mon Sep 17 00:00:00 2001 From: Maciej Sobkowiak Date: Tue, 15 Feb 2022 04:03:45 +0100 Subject: [PATCH] Working on setup script --- .gitignore | 2 +- README.md | 17 +++++++++++++++++ src/__import__.py | 2 ++ src/consts.py | 10 ++++++++++ src/generators.py | 41 ++++++++++++++++++++++++++++++++++++++++ src/helpers.py | 46 +++++++++++++++++++++++++++++++++++++++++++++ src/setup.py | 29 ++++++++++++++++++++++++++++ src/utils.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 README.md create mode 100644 src/__import__.py create mode 100644 src/consts.py create mode 100644 src/generators.py create mode 100644 src/helpers.py create mode 100644 src/setup.py create mode 100644 src/utils.py diff --git a/.gitignore b/.gitignore index fcbe59a..2d5a4f7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ venv/ **/images* **/data/train_features* **/data/train_labels* - +**/__pychache__* diff --git a/README.md b/README.md new file mode 100644 index 0000000..436b85c --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# Setup +W przypadku rosterio i GDAL należy pobrać wheele z repozytorium [tutaj](https://www.lfd.uci.edu/~gohlke/pythonlibs/), a nastepnie zainstalować je lokalnie. Reszta paczek zainstaluje się automatycznie komendą: +``` +pip install -r requirements.txt +``` +Pierwsze uruchomienie polega na uruchomieniu skryptu setup.py +``` +python3 setup.py +``` +Skrypt ten sprawi że zdjęcia zostaną przekonwertowane z formatu tif na format jpg. + +#TODO + +Jeżeli chcemy zwizualizować przekonwertowane zdjęcia, należy urochomić skrypt z flagą --show +``` +python3 setup.py --show +``` \ No newline at end of file diff --git a/src/__import__.py b/src/__import__.py new file mode 100644 index 0000000..503a4f6 --- /dev/null +++ b/src/__import__.py @@ -0,0 +1,2 @@ +from consts import * +from helpers import * \ No newline at end of file diff --git a/src/consts.py b/src/consts.py new file mode 100644 index 0000000..ec0f1e6 --- /dev/null +++ b/src/consts.py @@ -0,0 +1,10 @@ +import pandas as pd +FEATURES ='../data/train_features' +LABELS = '../data/train_labels' +JPG_IMAGES = '../images' +FC_DIR = "fc" +METADATA = pd.read_csv('../data/On_Cloud_N_Cloud_Cover_Detection_Challenge_-_train_metadata.csv.csv') + +BATCH = 8 +IMG_SIZE = (512,512) +SEED = 7 \ No newline at end of file diff --git a/src/generators.py b/src/generators.py new file mode 100644 index 0000000..216f3ba --- /dev/null +++ b/src/generators.py @@ -0,0 +1,41 @@ +train_datagen = ImageDataGenerator(rescale=1 / 255.0, + horizontal_flip=True, + vertical_flip=True, + validation_split=0.2) + +# mask_datagen = ImageDataGenerator(rescale=1/255, +# validation_split=0.2) + +#training data +#rgb images +rgb_dir = '../input/ai4earth-mask-the-clouds/RGB_images' +rgb_gen = train_datagen.flow_from_directory(directory=rgb_dir, + target_size= img_size, + batch_size=BATCH, + class_mode=None, + classes=None, + shuffle=False, + seed=seed, + subset='training') + +#false color (nir,green,blue) +fc_dir = '../input/ai4earth-mask-the-clouds/False_color' +fc_gen = train_datagen.flow_from_directory( + directory=fc_dir, + target_size= img_size, + batch_size=BATCH, + class_mode=None, + classes=None, + shuffle=False,seed=seed, + subset='training') + +#training labels +mask_dir ='../input/ai4earth-mask-the-clouds/Masks' +mask_gen = train_datagen.flow_from_directory( + directory=mask_dir, + target_size= img_size, + batch_size=BATCH, + class_mode=None, + classes=None, + shuffle=False,seed=seed, + subset='training') \ No newline at end of file diff --git a/src/helpers.py b/src/helpers.py new file mode 100644 index 0000000..71a95aa --- /dev/null +++ b/src/helpers.py @@ -0,0 +1,46 @@ +import cv2 +import os +import random +import numpy as np +# import tensorflow as tf +import rasterio as rio +import matplotlib.pyplot as plt + +def load_img(path,expand_dim=False): + img = cv2.imread(path) +# img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB) + img = img / 255.0 + if expand_dim: + img = np.expand_dims(img,axis=0) + + return img + +def create_folder(name, path): + n = os.path.join(path, name) + if not os.path.exists(n): + os.makedirs(n) + return n + +def scale(band): + return band/np.max(band) + +def convert_tif_to_jpg(rasters_dir, + rgb_path=None, + false_color_path=None): + + b2 = scale(rio.open(rasters_dir+'/B02.tif').read().reshape(512, 512, 1)) + b3 = scale(rio.open(rasters_dir+'/B03.tif').read().reshape(512, 512, 1)) + b4 = scale(rio.open(rasters_dir+'/B04.tif').read().reshape(512, 512, 1)) + b8 = scale(rio.open(rasters_dir+'/B08.tif').read().reshape(512, 512, 1)) + + file_name = rasters_dir.split(os.sep)[-1] + + rgb = np.dstack([b4, b3, b2]) + + plt.imsave(fname=rgb_path + f'/{file_name}.jpeg', + arr=rgb) + + fc = np.dstack([b8, b3, b2]) + + plt.imsave(fname=false_color_path + f'/{file_name}.jpeg', + arr=fc) diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 0000000..4e291e9 --- /dev/null +++ b/src/setup.py @@ -0,0 +1,29 @@ +import os +import numpy as np +import pandas as pd +import random +import matplotlib.pyplot as plt +import time + +#progess bar +from tqdm import tqdm + +import cv2 +import warnings +#deep learning + +from consts import SEED, JPG_IMAGES, FC_DIR, FEATURES +from helpers import create_folder, convert_tif_to_jpg + +warnings.filterwarnings('ignore') + +def transform_photo(tif_dir): + dp = create_folder(tif_dir, JPG_IMAGES) + fc = create_folder(FC_DIR, dp) + convert_tif_to_jpg(os.path.join(FEATURES, tif_dir), dp, fc) + + +if __name__ == "__main__": + for d in os.listdir(FEATURES): + print(str(d), end='\t') + transform_photo(d) \ No newline at end of file diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 0000000..b9986ce --- /dev/null +++ b/src/utils.py @@ -0,0 +1,48 @@ +def plot_image_grid(image_list, + label_list, + sample_images=False, + num_images=6, + pre_title='class', + num_img_per_row=3, + cmap=None, + img_h_w=3): + '''viz images from a list of images and labels + INPUTS: + image_list: a list of images to be plotted, + label_list: a list of correspomding image labels''' + + + + #number of img rows + n_row= num_images//num_img_per_row + + plt.subplots(n_row,num_img_per_row,figsize=(img_h_w*num_img_per_row,img_h_w*n_row)) + + if sample_images: + #select_random images + sampled_ids = random.choices(np.arange(0,len(image_list)),k=num_images) + + for i,idx in enumerate(sampled_ids): + + img = image_list[idx] + label = label_list[i] + plt.subplot(n_row,num_img_per_row,i+1) + plt.title(f'{pre_title} - {label}') + plt.axis('off') + plt.imshow(img,cmap=cmap) + else: + for i,img in enumerate(image_list): + + label = label_list[i] + plt.subplot(n_row,num_img_per_row,i+1) + plt.title(f'{pre_title} - {label}') + plt.axis('off') + plt.imshow(img,cmap=cmap) + + # break the loop + if i==num_images-1 : + break + + #show + plt.tight_layout() + plt.show() \ No newline at end of file