Working on setup script

.gitignore

@@ -2,4 +2,4 @@ venv/
 **/images*
 **/data/train_features*
 **/data/train_labels*
-
+**/__pychache__*

README.md

@@ -0,0 +1,17 @@
+# Setup
+W przypadku rosterio i GDAL należy pobrać wheele z repozytorium [tutaj](https://www.lfd.uci.edu/~gohlke/pythonlibs/), a nastepnie zainstalować je lokalnie. Reszta paczek zainstaluje się automatycznie komendą:
+```
+pip install -r requirements.txt
+```
+Pierwsze uruchomienie polega na uruchomieniu skryptu setup.py
+```
+python3 setup.py
+```
+Skrypt ten sprawi że zdjęcia zostaną przekonwertowane z formatu tif na format jpg.
+
+#TODO
+
+Jeżeli chcemy zwizualizować przekonwertowane zdjęcia, należy urochomić skrypt z flagą --show
+```
+python3 setup.py --show
+```

src/__import__.py

@@ -0,0 +1,2 @@
+from consts import *
+from helpers import *

src/consts.py

@@ -0,0 +1,10 @@
+import pandas as pd
+FEATURES ='../data/train_features'
+LABELS  = '../data/train_labels'
+JPG_IMAGES = '../images'
+FC_DIR = "fc"
+METADATA = pd.read_csv('../data/On_Cloud_N_Cloud_Cover_Detection_Challenge_-_train_metadata.csv.csv')
+
+BATCH = 8
+IMG_SIZE = (512,512)
+SEED = 7

src/generators.py

@@ -0,0 +1,41 @@
+train_datagen = ImageDataGenerator(rescale=1 / 255.0,
+                                    horizontal_flip=True,
+                                    vertical_flip=True,
+                                  validation_split=0.2)
+
+# mask_datagen =  ImageDataGenerator(rescale=1/255,
+#                                   validation_split=0.2)
+
+#training data
+#rgb images
+rgb_dir = '../input/ai4earth-mask-the-clouds/RGB_images'
+rgb_gen = train_datagen.flow_from_directory(directory=rgb_dir,
+                                           target_size= img_size,
+                                           batch_size=BATCH,
+                                           class_mode=None,
+                                           classes=None,
+                                           shuffle=False,
+                                            seed=seed,
+                                           subset='training')
+
+#false color (nir,green,blue)
+fc_dir = '../input/ai4earth-mask-the-clouds/False_color'
+fc_gen = train_datagen.flow_from_directory(
+                                          directory=fc_dir,
+                                           target_size= img_size,
+                                           batch_size=BATCH,
+                                           class_mode=None,
+                                           classes=None,
+                                           shuffle=False,seed=seed,
+                                            subset='training')
+
+#training labels
+mask_dir ='../input/ai4earth-mask-the-clouds/Masks'
+mask_gen = train_datagen.flow_from_directory(
+                                          directory=mask_dir,
+                                           target_size= img_size,
+                                           batch_size=BATCH,
+                                           class_mode=None,
+                                           classes=None,
+                                           shuffle=False,seed=seed,
+                                           subset='training')

src/helpers.py

@@ -0,0 +1,46 @@
+import cv2
+import os
+import random
+import numpy as np
+# import tensorflow as tf
+import rasterio as rio
+import matplotlib.pyplot as plt
+
+def load_img(path,expand_dim=False):
+    img = cv2.imread(path)
+#   img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
+    img = img / 255.0    
+    if expand_dim:
+        img = np.expand_dims(img,axis=0)
+        
+    return img
+
+def create_folder(name, path):
+    n = os.path.join(path, name)
+    if not os.path.exists(n):
+        os.makedirs(n)
+    return n
+
+def scale(band):
+    return band/np.max(band)
+
+def convert_tif_to_jpg(rasters_dir,
+                            rgb_path=None,
+                            false_color_path=None):
+
+    b2 = scale(rio.open(rasters_dir+'/B02.tif').read().reshape(512, 512, 1))
+    b3 = scale(rio.open(rasters_dir+'/B03.tif').read().reshape(512, 512, 1))
+    b4 = scale(rio.open(rasters_dir+'/B04.tif').read().reshape(512, 512, 1))
+    b8 = scale(rio.open(rasters_dir+'/B08.tif').read().reshape(512, 512, 1))
+
+    file_name = rasters_dir.split(os.sep)[-1]
+
+    rgb = np.dstack([b4, b3, b2])
+
+    plt.imsave(fname=rgb_path + f'/{file_name}.jpeg',
+               arr=rgb)
+
+    fc = np.dstack([b8, b3, b2])
+
+    plt.imsave(fname=false_color_path + f'/{file_name}.jpeg',
+               arr=fc)

src/setup.py

@@ -0,0 +1,29 @@
+import os
+import numpy as np
+import pandas as pd
+import random
+import matplotlib.pyplot as plt
+import time
+
+#progess bar 
+from tqdm import tqdm
+
+import cv2 
+import warnings 
+#deep learning
+
+from consts import SEED, JPG_IMAGES, FC_DIR, FEATURES
+from helpers import  create_folder, convert_tif_to_jpg
+
+warnings.filterwarnings('ignore')
+
+def transform_photo(tif_dir):
+    dp = create_folder(tif_dir, JPG_IMAGES)
+    fc = create_folder(FC_DIR, dp)
+    convert_tif_to_jpg(os.path.join(FEATURES, tif_dir), dp, fc)
+
+
+if __name__ == "__main__":
+    for d in os.listdir(FEATURES):
+        print(str(d), end='\t')
+        transform_photo(d)

src/utils.py

@@ -0,0 +1,48 @@
+def plot_image_grid(image_list,
+                    label_list,
+                    sample_images=False,
+                    num_images=6,
+                    pre_title='class',
+                    num_img_per_row=3,
+                    cmap=None,
+                    img_h_w=3):
+    '''viz images from a list of images and labels
+    INPUTS:
+    image_list: a list of images to be plotted,
+    label_list: a list of correspomding image labels'''
+    
+
+
+    #number of img rows
+    n_row= num_images//num_img_per_row
+
+    plt.subplots(n_row,num_img_per_row,figsize=(img_h_w*num_img_per_row,img_h_w*n_row))
+
+    if sample_images:
+    #select_random images 
+        sampled_ids = random.choices(np.arange(0,len(image_list)),k=num_images)
+
+        for i,idx in enumerate(sampled_ids):
+
+            img = image_list[idx]
+            label = label_list[i]
+            plt.subplot(n_row,num_img_per_row,i+1)
+            plt.title(f'{pre_title} - {label}')
+            plt.axis('off')
+            plt.imshow(img,cmap=cmap)
+    else:
+        for i,img in enumerate(image_list):
+
+            label = label_list[i]
+            plt.subplot(n_row,num_img_per_row,i+1)
+            plt.title(f'{pre_title} - {label}')
+            plt.axis('off')
+            plt.imshow(img,cmap=cmap)
+
+            # break the loop 
+            if i==num_images-1 :
+                  break 
+
+    #show
+    plt.tight_layout()
+    plt.show()