diff --git a/kelner/main.py b/kelner/main.py index b7c478d..21bac9e 100644 --- a/kelner/main.py +++ b/kelner/main.py @@ -9,6 +9,9 @@ from kelner.src.managers.TableManager import TableManager from kelner.src.managers.WaiterManager import WaiterManager from kelner.src.algorithms.DecisionTree import Tree_Builder from kelner.src.managers.KitchenManager import KitchenManager +from kelner.src.algorithms.CNN.PrepareData import LoadModelThread +import kelner.src.settings as settings +import time # create screen consts Scale = 2 # scale for all images used within project @@ -18,10 +21,20 @@ GridCountX = 15 # number of columns in grid GridCountY = 9 # number of rows in grid ScreenWidth = CellSize * GridCountX + 2 * PaintOffset # screen width in pixels ScreenHeight = CellSize * GridCountY + 2 * PaintOffset # screen height in pixels +running_tasks = {'table': [], 'waiter': []} # initialize background gridBoard = GridBoard(ScreenWidth, ScreenHeight) +# start loading prediction model +settings.init() +load_model_thread = LoadModelThread() + +load_model_thread.start() +# joining this thread to main thread. Man thread will be started after this finish +load_model_thread.join() + + # initialize drawable objects manager drawableManager = DrawableCollection() @@ -76,12 +89,10 @@ drawableManager.add(waiter1) # drawableManager.add(waiter4) -# TODO: create kitchen kitchen = Kitchen(5, GridCountX - 5, 5, GridCountY - 5, CellSize, PaintOffset) drawableManager.add(kitchen) kitchenManager = KitchenManager(drawableManager, gridBoard) - # My comment # initialize a number of tables given in range for i in range(0, 40): @@ -92,10 +103,12 @@ for i in range(0, 40): # new thread controlling tables tableTask = TableManager(drawableManager, menuManager) tableTask.start() +running_tasks['table'].append(tableTask) # new thread controlling waiter -waiter1Task = WaiterManager(drawableManager, [waiter1], kitchenManager) +waiter1Task = WaiterManager(drawableManager, [waiter1], kitchenManager, menuManager, tableTask) waiter1Task.start() +running_tasks['waiter'].append(tableTask) # waiter2Task = WaiterManager(drawableManager, [waiter2]) # waiter2Task.start() diff --git a/kelner/src/algorithms/CNN/ModelPrediction.py b/kelner/src/algorithms/CNN/ModelPrediction.py new file mode 100644 index 0000000..45f8a38 --- /dev/null +++ b/kelner/src/algorithms/CNN/ModelPrediction.py @@ -0,0 +1,40 @@ +from tensorflow.keras.preprocessing import image +import tensorflow as tf +from tensorflow.keras.models import load_model +import numpy as np +import kelner.src.settings as settings + + + +class Predictor: + def __init__(self, food_list): + self._food_list = food_list + self._model = settings.tensorflowModel + + def predict_classes(self, images): + for img in images: + img = image.load_img(img, target_size=(224, 224)) + img = image.img_to_array(img) + img = np.expand_dims(img, axis=0) + img /= 255. + + pred = self._model.predict(img) + index = np.argmax(pred) + self._food_list.sort() + pred_value = self._food_list[index] + print("Pred: {}, index: {}, pred_value:{}".format(pred, index, pred_value)) + print("THIS IS A:{}".format(pred_value)) + + def predict_class(self, img): + + img = image.load_img(img, target_size=(224, 224)) + img = image.img_to_array(img) + img = np.expand_dims(img, axis=0) + img /= 255. + + pred = self._model.predict(img) + index = np.argmax(pred) + self._food_list.sort() + pred_value = self._food_list[index] + # print("Pred: {}, index: {}, pred_value:{}".format(pred, index, pred_value)) + print("THIS IS A:{}".format(pred_value)) diff --git a/kelner/src/algorithms/CNN/PrepareData.py b/kelner/src/algorithms/CNN/PrepareData.py new file mode 100644 index 0000000..cbfc609 --- /dev/null +++ b/kelner/src/algorithms/CNN/PrepareData.py @@ -0,0 +1,110 @@ +from collections import defaultdict +from shutil import copytree, rmtree, copy +import matplotlib.pyplot as plt +import os +os.environ['TF_CPP_MIN_LOG_LEVEL']='2' +import threading +from tensorflow.keras.models import load_model +import kelner.src.settings as settings + +# currently all images are not stored in repo because of big weight (5 GB) +data_dir = 'D:\\Nauka\\Studia\\4_sem\\SztucznaInteligencja\\A_star\\CNN\\foodRecognitionCNN\\food-101\\images' +folder_dir = 'D:\\Nauka\\Studia\\4_sem\\SztucznaInteligencja\\A_star\\CNN\\foodRecognitionCNN\\food-101' +foods_sorted = sorted(os.listdir(data_dir)) +food_id = 0 + + +# VISUALIZE DATA # +#################################################################### +# rows = 17 +# cols = 6 +# fig, ax = plt.subplots(rows, cols, figsize=(50, 50)) +# fig.suptitle("Showing one random image from each class", y=1.05, fontsize=24) + +# for i in range(rows): +# for j in range(cols): +# try: +# food_selected = foods_sorted[food_id] +# food_id += 1 +# except: +# break +# if food_selected == '.DS_Store': +# continue +# +# food_selected_images = os.listdir(os.path.join(data_dir, food_selected)) +# food_selected_random = np.random.choice(food_selected_images) +# img = plt.imread(os.path.join(data_dir, food_selected, food_selected_random)) +# ax[i][j].imshow(img) +# ax[i][j].set_title(food_selected, pad=10) +# +# plt.setp(ax, xticks=[], yticks=[]) +# plt.tight_layout() + +def prepare_data(filepath, source, dest): + print('Creating test data...') + print("filepath{} source{} dest{}".format(filepath, source, dest)) + class_images = defaultdict(list) + with open(filepath, 'r') as txt: + paths = [read.strip() for read in txt.readlines()] + for p in paths: + food = p.split('/') + class_images[food[0]].append(food[1] + '.jpg') + + for food in class_images.keys(): + print("\nCopying images into ", food) + if not os.path.exists(os.path.join(dest, food)): + os.makedirs(os.path.join(dest, food)) + for i in class_images[food]: + copy(os.path.join(source, food, i), os.path.join(dest, food, i)) + print('Done copying') + + +def count_files(dir): + files_len = 0 + for base, dirs, files in os.walk(dir): + for file in files: + files_len += 1 + return files_len + + +# create train_mini and test_mini dataset +def dataset_mini(food_list, src, dest): + if os.path.exists(dest): + rmtree(dest) + os.makedirs(dest) + for food_item in food_list: + # recursive copying all subdirectories + copytree(os.path.join(src, food_item), os.path.join(dest, food_item)) + + +food_list = ['pizza', 'apple_pie', 'donuts', 'sushi', 'omelette', 'nachos', 'tiramisu', 'pho', 'carrot_cake', 'mussels', + 'waffles', 'hot_dog', 'hamburger'] +src_train = os.path.abspath('train') +dest_train = os.path.abspath('train_mini') +src_test = os.path.abspath('test') +dest_test = os.path.abspath('test_mini') + +# # create minisets +# print(os.path.join(src_train)) +# print('Creating train_mini folder') +# dataset_mini(food_list, src_train, dest_train) +# print('Creating test_mini folder') +# dataset_mini(food_list, src_test, dest_test) +# +# # count minisets +# print("Total data in mini_train folder") +# print(count_files(dest_train)) +# print("Total data in mini_test folder") +# print(count_files(dest_test)) + +class LoadModelThread(threading.Thread): + def __init__(self): + super().__init__() + self.__runThread = True + + def run(self): + model_path = 'D:\\Nauka\\Studia\\4_sem\\SztucznaInteligencja\\ProjektAI\\kelner\\src\\algorithms\\CNN\\trainedModels\\big_model_trained_3class.hdf5' + model = load_model(model_path) + settings.tensorflowModel = model + print("---MODEL LOADED---") + diff --git a/kelner/src/algorithms/CNN/TrainModel.py b/kelner/src/algorithms/CNN/TrainModel.py new file mode 100644 index 0000000..ab45b53 --- /dev/null +++ b/kelner/src/algorithms/CNN/TrainModel.py @@ -0,0 +1,88 @@ +import tensorflow as tf +import numpy as np +from collections import defaultdict +import collections +import tensorflow.keras.backend as K +import os +import random +import tensorflow as tf +import tensorflow.keras.backend as K +from tensorflow.keras import regularizers +from tensorflow.keras.applications.inception_v3 import InceptionV3 +from tensorflow.keras.models import Sequential, Model +from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten +from tensorflow.keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D +from tensorflow.keras.preprocessing.image import ImageDataGenerator +from tensorflow.keras.callbacks import ModelCheckpoint, CSVLogger +from tensorflow.keras.optimizers import SGD +from tensorflow.keras.regularizers import l2 +from tensorflow import keras +from tensorflow.keras import models +import cv2 +from PIL import Image + + +### NOT TESTED IN PYCHARM ### +### CHECK JUPYTER NOTEBOOK FILE ### + +# using model pretrainded on datasets like ImageNet +n_classes = 13 +IMGSIZE = 224 +training_data_dir = os.path.abspath('train_mini') +validation_data_dir = os.path.abspath('test_mini') +number_train_files = 9750 # 75750 +number_test_files = 3250 # 25250 +batch_size = 16 + +# Normalize images to avoid repeatability of some marks on images +train_datagen = ImageDataGenerator( + # rescale all pixels from image + rescale=1. / 255, + shear_range=0.2, + zoom_range=0.2, + horizontal_flip=True +) + +test_datagen = ImageDataGenerator(rescale=1. / 255) + +train_generator = train_datagen.flow_from_directory( + training_data_dir, + target_size=(IMGSIZE, IMGSIZE), + batch_size=batch_size, + class_mode='categorical' +) + +validation_generator = test_datagen.flow_from_directory( + validation_data_dir, + target_size=(IMGSIZE, IMGSIZE), + batch_size=batch_size, + class_mode='categorical' +) + +inception = InceptionV3(weights='imagenet', include_top=False) +x = inception.output +x = GlobalAveragePooling2D()(x) # reduce spatial dimensions of layer, prevent overfitting +x = Dense(128, activation='relu')(x) # dense layer with relu activation function +x = Dropout(0.2)(x) # prevent overfitting + +predictions = Dense(13, kernel_regularizer=regularizers.l2(0.005), activation='softmax')(x) + +model = Model(inputs=inception.input, outputs=predictions) +# same as model.fit +# takes 2 arguments - optimizer and loss +model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy']) + +# save best prediction +checkpointer = ModelCheckpoint(filepath=os.path.abspath('pycharm_trained_model.hdf5'), verbose=1, save_best_only=True) +# log history +csv_logger = CSVLogger('history_3class.log') + +history = model.fit_generator(train_generator, + steps_per_epoch=number_train_files // batch_size, + validation_data=validation_generator, + validation_steps=number_test_files // batch_size, + epochs=30, + verbose=1, + callbacks=[csv_logger, checkpointer]) + +model.save("big_model_trained_3class.hdf5") diff --git a/kelner/src/algorithms/CNN/trainedModels/big_model_trained_3class.hdf5 b/kelner/src/algorithms/CNN/trainedModels/big_model_trained_3class.hdf5 new file mode 100644 index 0000000..820db8a Binary files /dev/null and b/kelner/src/algorithms/CNN/trainedModels/big_model_trained_3class.hdf5 differ diff --git a/kelner/src/components/GridBoard.py b/kelner/src/components/GridBoard.py index 723d5a9..f6b7160 100644 --- a/kelner/src/components/GridBoard.py +++ b/kelner/src/components/GridBoard.py @@ -31,9 +31,5 @@ class GridBoard: def udpdate(self): pygame.display.update() - def update_and_sleep(self, sec): - pygame.display.update() - time.sleep(sec) - def get_screen(self): return self.__screen \ No newline at end of file diff --git a/kelner/src/components/Kitchen.py b/kelner/src/components/Kitchen.py index 0d53422..75e9060 100644 --- a/kelner/src/components/Kitchen.py +++ b/kelner/src/components/Kitchen.py @@ -22,7 +22,7 @@ class Kitchen(Drawable): self._preparing_orders = [] for order in orders.items(): self._preparing_orders.append(order) - print("Added {} to kitchen".format(order)) + # print("Added {} to kitchen".format(order)) def get_ready_orders(self): print("Ready orders on kitchen: {}".format(self._ready_orders)) @@ -80,18 +80,17 @@ class Kitchen(Drawable): def draw_order(self, dishes, screen): draw_screen = screen.get_screen() + img_paths = [] for i, dish in enumerate(dishes): - img_path = str(dish) + '/' + str(random.randint(0, 4)) + '.jpg' - print("Image drawing: {}".format(img_path)) - image = self.getImage(Images.Dishes, img_path) + img_path = os.path.join(str(dish), str(random.randint(0, 4)) + '.jpg') + img_path = os.path.join(os.getcwd(), 'images', 'testDishes', img_path) size = int(self.getCellSize()) - print(os.getcwd()) - image = pygame.transform.scale((pygame.image.load(os.path.join(os.getcwd(), 'images', 'testDishes', img_path))), (size, size)) - print((13 - i) * self.getCellSize(), 1 * self.getCellSize()) - draw_screen.blit(image, ((13 - i) * self.getCellSize(), 1 * self.getCellSize())) + image = pygame.transform.scale((pygame.image.load(img_path)), (size, size)) + draw_screen.blit(image, ((12 - i) * self.getCellSize(), 1 * self.getCellSize())) screen.udpdate() - - # screen.update_and_sleep(5) + img_paths.append(str(img_path)) + # print("Image drawing: {}".format(img_path)) + return img_paths diff --git a/kelner/src/managers/DrawableCollection.py b/kelner/src/managers/DrawableCollection.py index 88c88d8..8046717 100644 --- a/kelner/src/managers/DrawableCollection.py +++ b/kelner/src/managers/DrawableCollection.py @@ -67,6 +67,11 @@ class DrawableCollection: kitchen = item return kitchen + def lock(self): + self.__waiterLock.acquire() + + def unlock(self): + self.__waiterLock.release() # gets all waiters from collection def getWaiters(self): diff --git a/kelner/src/managers/KitchenManager.py b/kelner/src/managers/KitchenManager.py index 357d7e5..29e4443 100644 --- a/kelner/src/managers/KitchenManager.py +++ b/kelner/src/managers/KitchenManager.py @@ -12,7 +12,6 @@ class KitchenManager(threading.Thread): self._gridboard = gridboard self.__runThread = False - def prepare_dish(self): pass @@ -24,34 +23,33 @@ class KitchenManager(threading.Thread): kitchen.clear_orders() return out - def pass_and_return_order(self, orders, kitchen): return kitchen.pass_and_return_order(orders) def draw_orders(self, orders, kitchen): if orders: - for order in orders: + for i, order in enumerate(orders): dishes = order[1] if dishes: - kitchen.draw_order(dishes, self._gridboard) + paths = kitchen.draw_order(dishes, self._gridboard) + print("Order nr{}: paths:" + "{}".format(i, paths)) + return paths - def run(self, orders, kitchen): + # TODO: recognize here + + def draw_single_order(self, order, kitchen): + dishes = order[1] + if dishes: + paths = kitchen.draw_order(dishes, self._gridboard) + # print("Ordering table: {} Order paths: {}".format(order[0], paths)) + return paths + + def run(self): self.__runThread = True - self.draw_orders(orders, kitchen) - time.sleep(10) def stop(self): self.__runThread = False def is_running(self): return self.__runThread - - - # def draw_orders(self, kitchen, orders, draw_manager): - # for order in orders: - # dishes = order[1] - # i = 0 - # print("DISHES TO PRINT: {}".format(dishes)) - - - # TODO: draw real images diff --git a/kelner/src/managers/MenuManager.py b/kelner/src/managers/MenuManager.py index 292bc27..d46df02 100644 --- a/kelner/src/managers/MenuManager.py +++ b/kelner/src/managers/MenuManager.py @@ -19,3 +19,6 @@ class MenuManager: order += [(self.__menuCard[random.randint(0, len(self.__menuCard) - 1)])] return order + def get_menu(self): + return self.__menuCard + diff --git a/kelner/src/managers/TableManager.py b/kelner/src/managers/TableManager.py index ce14258..5d6b9ea 100644 --- a/kelner/src/managers/TableManager.py +++ b/kelner/src/managers/TableManager.py @@ -27,3 +27,6 @@ class TableManager(threading.Thread): def stop(self): self.__runThread = False + + def resume(self): + self.__runThread = True diff --git a/kelner/src/managers/WaiterManager.py b/kelner/src/managers/WaiterManager.py index a969197..5322e95 100644 --- a/kelner/src/managers/WaiterManager.py +++ b/kelner/src/managers/WaiterManager.py @@ -4,17 +4,21 @@ import sys from kelner.src.components.Table import Status from kelner.src.algorithms.AStar.Finder import Finder from kelner.src.algorithms.BFS.BFS import BFS +from kelner.src.components.Table import Status +from kelner.src.algorithms.CNN.ModelPrediction import Predictor # creates new thread class WaiterManager(threading.Thread): - def __init__(self, drawableManager, waiters, kitchenManager): + def __init__(self, drawableManager, waiters, kitchen_manager, menu_manager, table_manager): super().__init__() self.__drawableManager = drawableManager self.__waiters = waiters self.__runThread = True - self._kitchen_manager = kitchenManager + self._kitchen_manager = kitchen_manager + self._menu_manager = menu_manager + self._table_manager = table_manager def __getNearestTargetPath(self, waiter, target): distance = sys.maxsize @@ -72,7 +76,6 @@ class WaiterManager(threading.Thread): turns = result lessTurnsTable = table - if lessTurnsTable is not None: tables.remove(lessTurnsTable) self.__changeWaiterDirection(waiter, lessTurnsTable.getX(), lessTurnsTable.getY()) @@ -104,7 +107,8 @@ class WaiterManager(threading.Thread): print("Ready to go") return ready_orders - + def predict_orders(self, orders, waiter): + pass # changes the status of a random table from NotReady to Ready @@ -126,36 +130,58 @@ class WaiterManager(threading.Thread): self.__changeWaiterDirection(waiter, step[0], step[1]) self.__moveWaiter(waiter, step[0], step[1]) + # check if waiter is near kitchen if (waiter.getX(), waiter.getY()) == (14, 1) or (waiter.getX(), waiter.getY()) == (13, 0): if waiter.get_target() == 'kitchen': kitchen = self.__drawableManager.get_kitchen() waiter_orders = waiter.getAcceptedOrders() - print(type(waiter_orders)) print("Waiter near kitchen. Collected orders: {}".format(waiter_orders)) kitchen.add_orders(waiter_orders) received_orders = kitchen.get_ready_orders() waiter.clear_accepted_orders() - if received_orders: # self._kitchen_manager.draw_orders(received_orders) - KM = self._kitchen_manager - KM.run(received_orders, kitchen) - KM.stop() - - + # paths = self._kitchen_manager.draw_orders(received_orders, kitchen) + food_list = self._menu_manager.get_menu() + ImagePredictor = Predictor(food_list) + for order in received_orders: + paths = self._kitchen_manager.draw_single_order(order, kitchen) + self._kitchen_manager.run() + self._table_manager.stop() + # print("Printed images paths: {}".format(paths)) + if paths is not None: + for img_path in paths: + ImagePredictor.predict_class(img_path) + self.__drawableManager.forceRepaint() + self._kitchen_manager.stop() + self._table_manager.resume() + self._table_manager.join() # TODO: recognize images - # TODO: choose proper tableo - # TODO: set path to tables, another target - print('should blit') - kitchen.clear_orders() - waiter.make_ready() - waiter.set_target('table') + all_tables = self.__drawableManager.getTables(Status.Waiting) + for table in all_tables: + # order = table.get_order() + # print("Table: {}\norder: {}".format(table, order)) + pass + + # TODO: choose proper tableo + + # TODO: set path to tables, another target + + time.sleep(5) + self._kitchen_manager.stop() + + kitchen.clear_orders() + waiter.make_ready() + waiter.set_target('table') else: self.__collectOrder(waiter) def stop(self): self.__runThread = False + + def resume(self): + self.__runThread = True diff --git a/kelner/src/settings.py b/kelner/src/settings.py new file mode 100644 index 0000000..050e113 --- /dev/null +++ b/kelner/src/settings.py @@ -0,0 +1,3 @@ +def init(): + global tensorflowModel + tensorflowModel = None \ No newline at end of file