printing out recognized orders, bug with table thread

kelner/main.py

@@ -9,6 +9,9 @@ from kelner.src.managers.TableManager import TableManager
 from kelner.src.managers.WaiterManager import WaiterManager
 from kelner.src.algorithms.DecisionTree import Tree_Builder
 from kelner.src.managers.KitchenManager import KitchenManager
+from kelner.src.algorithms.CNN.PrepareData import LoadModelThread
+import kelner.src.settings as settings
+import time
 
 # create screen consts
 Scale = 2  # scale for all images used within project
@@ -18,10 +21,20 @@ GridCountX = 15  # number of columns in grid
 GridCountY = 9  # number of rows in grid
 ScreenWidth = CellSize * GridCountX + 2 * PaintOffset  # screen width in pixels
 ScreenHeight = CellSize * GridCountY + 2 * PaintOffset  # screen height in pixels
+running_tasks = {'table': [], 'waiter': []}
 
 # initialize background
 gridBoard = GridBoard(ScreenWidth, ScreenHeight)
 
+# start loading prediction model
+settings.init()
+load_model_thread = LoadModelThread()
+
+load_model_thread.start()
+# joining this thread to main thread. Man thread will be started after this finish
+load_model_thread.join()
+
+
 # initialize drawable objects manager
 drawableManager = DrawableCollection()
 
@@ -76,12 +89,10 @@ drawableManager.add(waiter1)
 # drawableManager.add(waiter4)
 
 
-# TODO: create kitchen
 kitchen = Kitchen(5, GridCountX - 5, 5, GridCountY - 5, CellSize, PaintOffset)
 drawableManager.add(kitchen)
 kitchenManager = KitchenManager(drawableManager, gridBoard)
 
-
 # My comment
 # initialize a number of tables given in range
 for i in range(0, 40):
@@ -92,10 +103,12 @@ for i in range(0, 40):
 # new thread controlling tables
 tableTask = TableManager(drawableManager, menuManager)
 tableTask.start()
+running_tasks['table'].append(tableTask)
 
 # new thread controlling waiter
-waiter1Task = WaiterManager(drawableManager, [waiter1], kitchenManager)
+waiter1Task = WaiterManager(drawableManager, [waiter1], kitchenManager, menuManager, tableTask)
 waiter1Task.start()
+running_tasks['waiter'].append(tableTask)
 
 # waiter2Task = WaiterManager(drawableManager, [waiter2])
 # waiter2Task.start()

kelner/src/algorithms/CNN/ModelPrediction.py

@@ -0,0 +1,40 @@
+from tensorflow.keras.preprocessing import image
+import tensorflow as tf
+from tensorflow.keras.models import load_model
+import numpy as np
+import kelner.src.settings as settings
+
+
+
+class Predictor:
+    def __init__(self, food_list):
+        self._food_list = food_list
+        self._model = settings.tensorflowModel
+
+    def predict_classes(self, images):
+        for img in images:
+            img = image.load_img(img, target_size=(224, 224))
+            img = image.img_to_array(img)
+            img = np.expand_dims(img, axis=0)
+            img /= 255.
+
+            pred = self._model.predict(img)
+            index = np.argmax(pred)
+            self._food_list.sort()
+            pred_value = self._food_list[index]
+            print("Pred: {}, index: {}, pred_value:{}".format(pred, index, pred_value))
+            print("THIS IS A:{}".format(pred_value))
+
+    def predict_class(self, img):
+
+        img = image.load_img(img, target_size=(224, 224))
+        img = image.img_to_array(img)
+        img = np.expand_dims(img, axis=0)
+        img /= 255.
+
+        pred = self._model.predict(img)
+        index = np.argmax(pred)
+        self._food_list.sort()
+        pred_value = self._food_list[index]
+        # print("Pred: {}, index: {}, pred_value:{}".format(pred, index, pred_value))
+        print("THIS IS A:{}".format(pred_value))

kelner/src/algorithms/CNN/PrepareData.py

@@ -0,0 +1,110 @@
+from collections import defaultdict
+from shutil import copytree, rmtree, copy
+import matplotlib.pyplot as plt
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
+import threading
+from tensorflow.keras.models import load_model
+import kelner.src.settings as settings
+
+# currently all images are not stored in repo because of big weight (5 GB)
+data_dir = 'D:\\Nauka\\Studia\\4_sem\\SztucznaInteligencja\\A_star\\CNN\\foodRecognitionCNN\\food-101\\images'
+folder_dir = 'D:\\Nauka\\Studia\\4_sem\\SztucznaInteligencja\\A_star\\CNN\\foodRecognitionCNN\\food-101'
+foods_sorted = sorted(os.listdir(data_dir))
+food_id = 0
+
+
+#                       VISUALIZE DATA                             #
+####################################################################
+# rows = 17
+# cols = 6
+# fig, ax = plt.subplots(rows, cols, figsize=(50, 50))
+# fig.suptitle("Showing one random image from each class", y=1.05, fontsize=24)
+
+# for i in range(rows):
+#     for j in range(cols):
+#         try:
+#             food_selected = foods_sorted[food_id]
+#             food_id += 1
+#         except:
+#             break
+#         if food_selected == '.DS_Store':
+#             continue
+#
+#         food_selected_images = os.listdir(os.path.join(data_dir, food_selected))
+#         food_selected_random = np.random.choice(food_selected_images)
+#         img = plt.imread(os.path.join(data_dir, food_selected, food_selected_random))
+#         ax[i][j].imshow(img)
+#         ax[i][j].set_title(food_selected, pad=10)
+#
+# plt.setp(ax, xticks=[], yticks=[])
+# plt.tight_layout()
+
+def prepare_data(filepath, source, dest):
+    print('Creating test data...')
+    print("filepath{} source{} dest{}".format(filepath, source, dest))
+    class_images = defaultdict(list)
+    with open(filepath, 'r') as txt:
+        paths = [read.strip() for read in txt.readlines()]
+        for p in paths:
+            food = p.split('/')
+            class_images[food[0]].append(food[1] + '.jpg')
+
+    for food in class_images.keys():
+        print("\nCopying images into ", food)
+        if not os.path.exists(os.path.join(dest, food)):
+            os.makedirs(os.path.join(dest, food))
+        for i in class_images[food]:
+            copy(os.path.join(source, food, i), os.path.join(dest, food, i))
+    print('Done copying')
+
+
+def count_files(dir):
+    files_len = 0
+    for base, dirs, files in os.walk(dir):
+        for file in files:
+            files_len += 1
+    return files_len
+
+
+# create train_mini and test_mini dataset
+def dataset_mini(food_list, src, dest):
+    if os.path.exists(dest):
+        rmtree(dest)
+    os.makedirs(dest)
+    for food_item in food_list:
+        # recursive copying all subdirectories
+        copytree(os.path.join(src, food_item), os.path.join(dest, food_item))
+
+
+food_list = ['pizza', 'apple_pie', 'donuts', 'sushi', 'omelette', 'nachos', 'tiramisu', 'pho', 'carrot_cake', 'mussels',
+             'waffles', 'hot_dog', 'hamburger']
+src_train = os.path.abspath('train')
+dest_train = os.path.abspath('train_mini')
+src_test = os.path.abspath('test')
+dest_test = os.path.abspath('test_mini')
+
+# # create minisets
+# print(os.path.join(src_train))
+# print('Creating train_mini folder')
+# dataset_mini(food_list, src_train, dest_train)
+# print('Creating test_mini folder')
+# dataset_mini(food_list, src_test, dest_test)
+#
+# # count minisets
+# print("Total data in mini_train folder")
+# print(count_files(dest_train))
+# print("Total data in mini_test folder")
+# print(count_files(dest_test))
+
+class LoadModelThread(threading.Thread):
+    def __init__(self):
+        super().__init__()
+        self.__runThread = True
+
+    def run(self):
+        model_path = 'D:\\Nauka\\Studia\\4_sem\\SztucznaInteligencja\\ProjektAI\\kelner\\src\\algorithms\\CNN\\trainedModels\\big_model_trained_3class.hdf5'
+        model = load_model(model_path)
+        settings.tensorflowModel = model
+        print("---MODEL LOADED---")
+

kelner/src/algorithms/CNN/TrainModel.py

@@ -0,0 +1,88 @@
+import tensorflow as tf
+import numpy as np
+from collections import defaultdict
+import collections
+import tensorflow.keras.backend as K
+import os
+import random
+import tensorflow as tf
+import tensorflow.keras.backend as K
+from tensorflow.keras import regularizers
+from tensorflow.keras.applications.inception_v3 import InceptionV3
+from tensorflow.keras.models import Sequential, Model
+from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten
+from tensorflow.keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D
+from tensorflow.keras.preprocessing.image import ImageDataGenerator
+from tensorflow.keras.callbacks import ModelCheckpoint, CSVLogger
+from tensorflow.keras.optimizers import SGD
+from tensorflow.keras.regularizers import l2
+from tensorflow import keras
+from tensorflow.keras import models
+import cv2
+from PIL import Image
+
+
+###     NOT TESTED IN PYCHARM   ###
+### CHECK JUPYTER NOTEBOOK FILE ###
+
+# using model pretrainded on datasets like ImageNet
+n_classes = 13
+IMGSIZE = 224
+training_data_dir = os.path.abspath('train_mini')
+validation_data_dir = os.path.abspath('test_mini')
+number_train_files = 9750  # 75750
+number_test_files = 3250  # 25250
+batch_size = 16
+
+# Normalize images to avoid repeatability of some marks on images
+train_datagen = ImageDataGenerator(
+    # rescale all pixels from image
+    rescale=1. / 255,
+    shear_range=0.2,
+    zoom_range=0.2,
+    horizontal_flip=True
+)
+
+test_datagen = ImageDataGenerator(rescale=1. / 255)
+
+train_generator = train_datagen.flow_from_directory(
+    training_data_dir,
+    target_size=(IMGSIZE, IMGSIZE),
+    batch_size=batch_size,
+    class_mode='categorical'
+)
+
+validation_generator = test_datagen.flow_from_directory(
+    validation_data_dir,
+    target_size=(IMGSIZE, IMGSIZE),
+    batch_size=batch_size,
+    class_mode='categorical'
+)
+
+inception = InceptionV3(weights='imagenet', include_top=False)
+x = inception.output
+x = GlobalAveragePooling2D()(x)  # reduce spatial dimensions of layer, prevent overfitting
+x = Dense(128, activation='relu')(x)  # dense layer with relu activation function
+x = Dropout(0.2)(x)  # prevent overfitting
+
+predictions = Dense(13, kernel_regularizer=regularizers.l2(0.005), activation='softmax')(x)
+
+model = Model(inputs=inception.input, outputs=predictions)
+# same as model.fit
+# takes 2 arguments - optimizer and loss
+model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy'])
+
+# save best prediction
+checkpointer = ModelCheckpoint(filepath=os.path.abspath('pycharm_trained_model.hdf5'), verbose=1, save_best_only=True)
+# log history
+csv_logger = CSVLogger('history_3class.log')
+
+history = model.fit_generator(train_generator,
+                              steps_per_epoch=number_train_files // batch_size,
+                              validation_data=validation_generator,
+                              validation_steps=number_test_files // batch_size,
+                              epochs=30,
+                              verbose=1,
+                              callbacks=[csv_logger, checkpointer])
+
+model.save("big_model_trained_3class.hdf5")

kelner/src/components/GridBoard.py

@@ -31,9 +31,5 @@ class GridBoard:
     def udpdate(self):
         pygame.display.update()
 
-    def update_and_sleep(self, sec):
-        pygame.display.update()
-        time.sleep(sec)
-
     def get_screen(self):
         return self.__screen

kelner/src/components/Kitchen.py

@@ -22,7 +22,7 @@ class Kitchen(Drawable):
             self._preparing_orders = []
         for order in orders.items():
             self._preparing_orders.append(order)
-            print("Added {} to kitchen".format(order))
+            # print("Added {} to kitchen".format(order))
 
     def get_ready_orders(self):
         print("Ready orders on kitchen: {}".format(self._ready_orders))
@@ -80,18 +80,17 @@ class Kitchen(Drawable):
 
     def draw_order(self, dishes, screen):
         draw_screen = screen.get_screen()
+        img_paths = []
         for i, dish in enumerate(dishes):
-            img_path = str(dish) + '/' + str(random.randint(0, 4)) + '.jpg'
-            print("Image drawing: {}".format(img_path))
-            image = self.getImage(Images.Dishes, img_path)
+            img_path = os.path.join(str(dish), str(random.randint(0, 4)) + '.jpg')
+            img_path = os.path.join(os.getcwd(), 'images', 'testDishes', img_path)
             size = int(self.getCellSize())
-            print(os.getcwd())
-            image = pygame.transform.scale((pygame.image.load(os.path.join(os.getcwd(), 'images', 'testDishes', img_path))), (size, size))
-            print((13 - i) * self.getCellSize(), 1 * self.getCellSize())
-            draw_screen.blit(image, ((13 - i) * self.getCellSize(), 1 * self.getCellSize()))
+            image = pygame.transform.scale((pygame.image.load(img_path)), (size, size))
+            draw_screen.blit(image, ((12 - i) * self.getCellSize(), 1 * self.getCellSize()))
             screen.udpdate()
-
-        # screen.update_and_sleep(5)
+            img_paths.append(str(img_path))
+            # print("Image drawing: {}".format(img_path))
+        return img_paths
 
 
 

kelner/src/managers/DrawableCollection.py

@@ -67,6 +67,11 @@ class DrawableCollection:
                 kitchen = item
         return kitchen
 
+    def lock(self):
+        self.__waiterLock.acquire()
+
+    def unlock(self):
+        self.__waiterLock.release()
 
     # gets all waiters from collection
     def getWaiters(self):

kelner/src/managers/KitchenManager.py

@@ -12,7 +12,6 @@ class KitchenManager(threading.Thread):
         self._gridboard = gridboard
         self.__runThread = False
 
-
     def prepare_dish(self):
         pass
 
@@ -24,34 +23,33 @@ class KitchenManager(threading.Thread):
         kitchen.clear_orders()
         return out
 
-
     def pass_and_return_order(self, orders, kitchen):
         return kitchen.pass_and_return_order(orders)
 
     def draw_orders(self, orders, kitchen):
         if orders:
-            for order in orders:
+            for i, order in enumerate(orders):
                 dishes = order[1]
                 if dishes:
-                    kitchen.draw_order(dishes, self._gridboard)
+                    paths = kitchen.draw_order(dishes, self._gridboard)
+                    print("Order nr{}: paths:"
+                          "{}".format(i, paths))
+        return paths
 
-    def run(self, orders, kitchen):
+        # TODO: recognize here
+
+    def draw_single_order(self, order, kitchen):
+        dishes = order[1]
+        if dishes:
+            paths = kitchen.draw_order(dishes, self._gridboard)
+            # print("Ordering table: {} Order paths: {}".format(order[0], paths))
+        return paths
+
+    def run(self):
         self.__runThread = True
-        self.draw_orders(orders, kitchen)
-        time.sleep(10)
 
     def stop(self):
         self.__runThread = False
 
     def is_running(self):
         return self.__runThread
-
-
-    # def draw_orders(self, kitchen, orders, draw_manager):
-    #     for order in orders:
-    #         dishes = order[1]
-    #         i = 0
-    #         print("DISHES TO PRINT: {}".format(dishes))
-
-
-        # TODO: draw real images

kelner/src/managers/MenuManager.py

@@ -19,3 +19,6 @@ class MenuManager:
             order += [(self.__menuCard[random.randint(0, len(self.__menuCard) - 1)])]
         return order
 
+    def get_menu(self):
+        return self.__menuCard
+

kelner/src/managers/TableManager.py

@@ -27,3 +27,6 @@ class TableManager(threading.Thread):
 
     def stop(self):
         self.__runThread = False
+
+    def resume(self):
+        self.__runThread = True

kelner/src/managers/WaiterManager.py

@@ -4,17 +4,21 @@ import sys
 from kelner.src.components.Table import Status
 from kelner.src.algorithms.AStar.Finder import Finder
 from kelner.src.algorithms.BFS.BFS import BFS
+from kelner.src.components.Table import Status
+from kelner.src.algorithms.CNN.ModelPrediction import Predictor
 
 
 # creates new thread
 class WaiterManager(threading.Thread):
 
-    def __init__(self, drawableManager, waiters, kitchenManager):
+    def __init__(self, drawableManager, waiters, kitchen_manager, menu_manager, table_manager):
         super().__init__()
         self.__drawableManager = drawableManager
         self.__waiters = waiters
         self.__runThread = True
-        self._kitchen_manager = kitchenManager
+        self._kitchen_manager = kitchen_manager
+        self._menu_manager = menu_manager
+        self._table_manager = table_manager
 
     def __getNearestTargetPath(self, waiter, target):
         distance = sys.maxsize
@@ -72,7 +76,6 @@ class WaiterManager(threading.Thread):
                     turns = result
                     lessTurnsTable = table
 
-
             if lessTurnsTable is not None:
                 tables.remove(lessTurnsTable)
                 self.__changeWaiterDirection(waiter, lessTurnsTable.getX(), lessTurnsTable.getY())
@@ -104,7 +107,8 @@ class WaiterManager(threading.Thread):
         print("Ready to go")
         return ready_orders
 
-
+    def predict_orders(self, orders, waiter):
+        pass
 
 
     # changes the status of a random table from NotReady to Ready
@@ -126,36 +130,58 @@ class WaiterManager(threading.Thread):
                         self.__changeWaiterDirection(waiter, step[0], step[1])
                         self.__moveWaiter(waiter, step[0], step[1])
 
+                    # check if waiter is near kitchen
                     if (waiter.getX(), waiter.getY()) == (14, 1) or (waiter.getX(), waiter.getY()) == (13, 0):
                         if waiter.get_target() == 'kitchen':
 
                             kitchen = self.__drawableManager.get_kitchen()
                             waiter_orders = waiter.getAcceptedOrders()
-                            print(type(waiter_orders))
                             print("Waiter near kitchen. Collected orders: {}".format(waiter_orders))
                             kitchen.add_orders(waiter_orders)
                             received_orders = kitchen.get_ready_orders()
                             waiter.clear_accepted_orders()
-
                             if received_orders:
                                 # self._kitchen_manager.draw_orders(received_orders)
-                                KM = self._kitchen_manager
-                                KM.run(received_orders, kitchen)
-                                KM.stop()
-
-
+                                # paths = self._kitchen_manager.draw_orders(received_orders,  kitchen)
+                                food_list = self._menu_manager.get_menu()
+                                ImagePredictor = Predictor(food_list)
 
+                                for order in received_orders:
+                                    paths = self._kitchen_manager.draw_single_order(order, kitchen)
+                                    self._kitchen_manager.run()
+                                    self._table_manager.stop()
+                                    # print("Printed images paths: {}".format(paths))
+                                    if paths is not None:
+                                        for img_path in paths:
+                                            ImagePredictor.predict_class(img_path)
+                                    self.__drawableManager.forceRepaint()
+                                self._kitchen_manager.stop()
+                                self._table_manager.resume()
+                                self._table_manager.join()
 
                                 # TODO: recognize images
-                                # TODO: choose proper tableo
-                                # TODO: set path to tables, another target
-                                print('should blit')
 
-                        kitchen.clear_orders()
-                        waiter.make_ready()
-                        waiter.set_target('table')
+                                all_tables = self.__drawableManager.getTables(Status.Waiting)
+                                for table in all_tables:
+                                    # order = table.get_order()
+                                    # print("Table: {}\norder: {}".format(table, order))
+                                    pass
+
+                                # TODO: choose proper tableo
+
+                                # TODO: set path to tables, another target
+
+                                time.sleep(5)
+                                self._kitchen_manager.stop()
+
+                            kitchen.clear_orders()
+                            waiter.make_ready()
+                            waiter.set_target('table')
                     else:
                         self.__collectOrder(waiter)
 
     def stop(self):
         self.__runThread = False
+
+    def resume(self):
+        self.__runThread = True

kelner/src/settings.py

@@ -0,0 +1,3 @@
+def init():
+    global tensorflowModel
+    tensorflowModel = None