Merge branch 'master' of https://git.wmi.amu.edu.pl/s444463/projekt_widzenie

README.md

@@ -1,5 +1,11 @@
 # projekt_widzenie
 
+## Autorzy
+
+Mikołaj Pokrywka,
+Kamil Guttmann,
+Andrzej Preibisz
+
 ## Run apllication
 1. `pip install -r requirements.txt`
 2. `sudo apt-get install ffmpeg`

training_model/train_eff.py

@@ -0,0 +1,74 @@
+# %% [markdown]
+# # Import the required libraries
+
+# %%
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+
+
+batch_size=32
+img_height=224
+img_width=224
+import os
+os.environ["CUDA_VISIBLE_DEVICES"]="0"
+
+train_ds = tf.keras.utils.image_dataset_from_directory(
+  "combined_data_skeleton/",
+  seed=123,
+  image_size=(img_height, img_width),
+  batch_size=batch_size)
+
+test_ds = tf.keras.utils.image_dataset_from_directory(
+  "test_data_own_skeleton",
+  seed=123,
+  image_size=(img_height, img_width),
+  batch_size=batch_size
+)
+
+class_names = train_ds.class_names
+print("Class names:",class_names)
+print("Total classes:",len(class_names))
+
+
+
+
+from keras import models, layers
+from tensorflow.keras.applications import EfficientNetB0
+
+img_size = [img_height, img_width]
+
+eff = EfficientNetB0(input_shape=img_size + [3], weights='imagenet', include_top=False)
+
+x = layers.Flatten()(eff.output)
+x = layers.Dense(len(class_names), activation='softmax')(x)
+
+model = models.Model(eff.input, x)
+
+# for layer in model.layers[:-1]:
+#     layer.trainable = False
+
+model.compile(loss='sparse_categorical_crossentropy',
+                     optimizer='rmsprop',
+                     metrics=['accuracy'])
+
+
+
+from tensorflow.keras.callbacks import EarlyStopping
+
+early_stopping = EarlyStopping(
+monitor="val_accuracy", patience=12,
+restore_best_weights=True,
+
+)
+
+
+history = model.fit(
+  train_ds,
+  batch_size=64,
+  validation_data=test_ds,
+  validation_batch_size=64,
+  epochs=50,
+  callbacks=[early_stopping]
+)
+model.save('effnet_sign_char_detection_model')

training_model/train_own_cnn.py

@@ -0,0 +1,131 @@
+# %% [markdown]
+# # Import the required libraries
+
+# %%
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+
+# %% [markdown]
+# # Preprocessing the image data
+
+# %%
+#taking the train validation ratio as 4:1
+
+# %%
+batch_size=32
+img_height=256
+img_width=256
+
+
+train_ds = tf.keras.utils.image_dataset_from_directory(
+  "asl_alphabet_train/asl_alphabet_train/",
+  validation_split=0.2,
+  subset="training",
+  seed=123,
+  image_size=(img_height, img_width),
+  batch_size=batch_size)
+
+# %%
+test_ds = tf.keras.utils.image_dataset_from_directory(
+  "asl_alphabet_train/asl_alphabet_train/",
+  validation_split=0.2,
+  subset="validation",
+  seed=123,
+  image_size=(img_height, img_width),
+  batch_size=batch_size)
+
+# %%
+class_names = train_ds.class_names
+print("Class names:",class_names)
+print("Total classes:",len(class_names))
+
+# %%
+#random samples of images from the train data
+
+# %%
+import matplotlib.pyplot as plt
+
+plt.figure(figsize=(10, 10))
+for images, labels in train_ds.take(1):
+  for i in range(29):
+    ax = plt.subplot(6,5 , i + 1)
+    plt.imshow(images[i].numpy().astype("uint8"))
+    plt.title(class_names[labels[i]])
+    plt.axis("off")
+
+# %% [markdown]
+# # Modelling and training
+
+# %%
+#modelling
+
+from tensorflow.keras import Sequential
+from tensorflow.keras import layers
+from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense
+
+model = Sequential([
+  layers.Rescaling(1./255, input_shape=(img_height, img_width, 3)),
+  layers.Conv2D(16, 3, padding='same', activation='relu'),
+  layers.MaxPooling2D(),
+  layers.Conv2D(32, 3, padding='same', activation='relu'),
+  layers.MaxPooling2D(),
+  layers.Conv2D(64, 3, padding='same', activation='relu'),
+  layers.MaxPooling2D(),
+  layers.Flatten(),
+  layers.Dense(128, activation='relu'),
+  layers.Dense(29,activation='softmax')
+])
+
+# %%
+model.summary()
+
+# %%
+model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',metrics=['accuracy'])
+
+# %%
+model.fit(train_ds, batch_size=128,validation_batch_size=128, validation_data=test_ds,epochs=20)
+
+
+model.save('sign_car_detection_model')
+
+
+
+# %% [markdown]
+# # Prediction on the test data
+
+# %%
+import os
+ 
+# Get the list of all files and directories
+path = "asl_alphabet_test/asl_alphabet_test/"
+dir_list = os.listdir(path)
+ 
+print(dir_list)
+
+# %%
+from sklearn.preprocessing import OneHotEncoder, LabelEncoder
+tf.keras.utils.load_img
+
+actual=[]
+pred=[]
+for i in dir_list:
+    actual.append(i.split('_')[0])
+    test_image = tf.keras.utils.load_img('asl_alphabet_test/asl_alphabet_test/'+i, target_size = (256, 256))
+    test_image = tf.keras.utils.img_to_array(test_image)
+    test_image = np.expand_dims(test_image, axis = 0)
+    result = model.predict(test_image)
+    pred.append(class_names[np.argmax(result)])
+
+# %%
+from sklearn.metrics import confusion_matrix, classification_report
+from sklearn.metrics import accuracy_score
+
+print("Test accuracy=",accuracy_score(pred,actual))
+print("Classification report:\n",classification_report(pred,actual))
+
+
+# %%
+
+
+

training_model/train_resnet.py

@@ -0,0 +1,74 @@
+# %% [markdown]
+# # Import the required libraries
+
+# %%
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+
+
+batch_size=64
+img_height=224
+img_width=224
+import os
+os.environ["CUDA_VISIBLE_DEVICES"]="1"
+
+train_ds = tf.keras.utils.image_dataset_from_directory(
+  "combined_data_skeleton/",
+  seed=123,
+  image_size=(img_height, img_width),
+  batch_size=batch_size)
+
+test_ds = tf.keras.utils.image_dataset_from_directory(
+  "test_data_own_skeleton",
+  seed=123,
+  image_size=(img_height, img_width),
+  batch_size=batch_size
+)
+
+class_names = train_ds.class_names
+print("Class names:",class_names)
+print("Total classes:",len(class_names))
+
+
+
+
+from keras import models, layers
+from tensorflow.keras.applications import ResNet101
+
+img_size = [img_height, img_width]
+
+res_model = ResNet101(input_shape=img_size + [3], weights='imagenet', include_top=False)
+
+x = layers.Flatten()(res_model.output)
+x = layers.Dense(len(class_names), activation='softmax')(x)
+
+model = models.Model(res_model.input, x)
+
+# for layer in model.layers[:-1]:
+#     layer.trainable = False
+
+model.compile(loss='sparse_categorical_crossentropy',
+                     optimizer='rmsprop',
+                     metrics=['accuracy'])
+
+
+
+from tensorflow.keras.callbacks import EarlyStopping
+
+early_stopping = EarlyStopping(
+monitor="val_accuracy", patience=12,
+restore_best_weights=True,
+
+)
+
+
+history = model.fit(
+  train_ds,
+  batch_size=64,
+  validation_data=test_ds,
+  validation_batch_size=64,
+  epochs=50,
+  callbacks=[early_stopping]
+)
+model.save('resnet101_sign_char_detection_model')

training_model/train_vgg16.py

@@ -0,0 +1,78 @@
+# %% [markdown]
+# # Import the required libraries
+
+# %%
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+
+
+batch_size=64
+img_height=224
+img_width=224
+import os
+os.environ["CUDA_VISIBLE_DEVICES"]="1"
+
+train_ds = tf.keras.utils.image_dataset_from_directory(
+  "combined_data_skeleton/",
+  seed=123,
+  image_size=(img_height, img_width),
+  batch_size=batch_size)
+
+test_ds = tf.keras.utils.image_dataset_from_directory(
+  "test_data_own_skeleton",
+  seed=123,
+  image_size=(img_height, img_width),
+  batch_size=batch_size
+)
+
+class_names = train_ds.class_names
+print("Class names:",class_names)
+print("Total classes:",len(class_names))
+
+
+
+
+from tensorflow.keras import Sequential
+from tensorflow.keras import layers
+from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense
+
+
+from keras import models, layers
+from tensorflow.keras.applications import VGG16
+
+img_size = [img_height, img_width]
+vgg_model = VGG16(input_shape=img_size + [3], weights='imagenet', include_top=False)
+
+x = layers.Flatten()(vgg_model.output)
+x = layers.Dense(len(class_names), activation='softmax')(x)
+
+model = models.Model(vgg_model.input, x)
+
+# for layer in model.layers[:-1]:
+#     layer.trainable = False
+
+model.compile(loss='sparse_categorical_crossentropy',
+                     optimizer='rmsprop',
+                     metrics=['accuracy'])
+
+
+
+from tensorflow.keras.callbacks import EarlyStopping
+
+early_stopping = EarlyStopping(
+monitor="val_accuracy", patience=12,
+restore_best_weights=True,
+
+)
+
+
+history = model.fit(
+  train_ds,
+  batch_size=128,
+  validation_data=test_ds,
+  validation_batch_size=128,
+  epochs=50,
+  callbacks=[early_stopping]
+)
+model.save('VGG16_sign_char_detection_model')

training_model/train_vgg19.py

@@ -0,0 +1,78 @@
+# %% [markdown]
+# # Import the required libraries
+
+# %%
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+
+
+batch_size=64
+img_height=224
+img_width=224
+import os
+os.environ["CUDA_VISIBLE_DEVICES"]="0"
+
+train_ds = tf.keras.utils.image_dataset_from_directory(
+  "combined_data_skeleton/",
+  seed=123,
+  image_size=(img_height, img_width),
+  batch_size=batch_size)
+
+test_ds = tf.keras.utils.image_dataset_from_directory(
+  "test_data_own_skeleton",
+  seed=123,
+  image_size=(img_height, img_width),
+  batch_size=batch_size
+)
+
+class_names = train_ds.class_names
+print("Class names:",class_names)
+print("Total classes:",len(class_names))
+
+
+
+
+from tensorflow.keras import Sequential
+from tensorflow.keras import layers
+from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense
+
+
+from keras import models, layers
+from tensorflow.keras.applications import VGG19
+
+img_size = [img_height, img_width]
+vgg_model = VGG19(input_shape=img_size + [3], weights='imagenet', include_top=False)
+
+x = layers.Flatten()(vgg_model.output)
+x = layers.Dense(len(class_names), activation='softmax')(x)
+
+model = models.Model(vgg_model.input, x)
+
+# for layer in model.layers[:-1]:
+#     layer.trainable = False
+
+model.compile(loss='sparse_categorical_crossentropy',
+                     optimizer='rmsprop',
+                     metrics=['accuracy'])
+
+
+
+from tensorflow.keras.callbacks import EarlyStopping
+
+early_stopping = EarlyStopping(
+monitor="val_accuracy", patience=12,
+restore_best_weights=True,
+
+)
+
+
+history = model.fit(
+  train_ds,
+  batch_size=128,
+  validation_data=test_ds,
+  validation_batch_size=128,
+  epochs=50,
+  callbacks=[early_stopping]
+)
+model.save('VGG19_sign_char_detection_model')