From 13e5c5d62cc06b5f4d2085624691a36c8e3c3667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksander=20Szama=C5=82ek?= Date: Fri, 27 May 2022 00:03:24 +0200 Subject: [PATCH] agent - item recognition loop --- ForkliftAgent.py | 5 +-- GameModel.py | 34 ++++++++++++++----- data/enum/ItemType.py | 6 ++-- ...imageClasification.py => Classificator.py} | 23 ++++++++----- imageClasification/how_to_use.py | 11 +++--- main.py | 6 ++-- util/PathByEnum.py | 7 ++-- visualization/DisplayItemListAttribute.py | 2 +- visualization/DisplayOrderList.py | 12 +++---- 9 files changed, 65 insertions(+), 41 deletions(-) rename imageClasification/{imageClasification.py => Classificator.py} (88%) diff --git a/ForkliftAgent.py b/ForkliftAgent.py index 8dee11e..d40448b 100644 --- a/ForkliftAgent.py +++ b/ForkliftAgent.py @@ -32,6 +32,7 @@ class ForkliftAgent(AgentBase): self.current_item = None self.item_station_completed = False self.provided_items: List[Item] = [] + self.ready_for_execution = False def queue_movement_actions(self, movement_actions: List[Action]): self.action_queue.extend(movement_actions) @@ -102,7 +103,7 @@ class ForkliftAgent(AgentBase): stations = dict(self.graph.packingStations) if i.real_type == ItemType.SHELF: packing_station = stations[PatchType.packingA] - elif i.real_type == ItemType.EGG: + elif i.real_type == ItemType.FRIDGE: packing_station = stations[PatchType.packingB] elif i.real_type == ItemType.DOOR: packing_station = stations[PatchType.packingC] @@ -147,7 +148,7 @@ class ForkliftAgent(AgentBase): def step(self) -> None: if len(self.action_queue) > 0: self.move() - elif len(self.orderList) > 0: + elif self.ready_for_execution and len(self.orderList) > 0: if (self.current_order is not None and len(self.current_order.items)) == 0: self.fulfilled_orders.append(self.current_order) self.current_order = None diff --git a/GameModel.py b/GameModel.py index 45999f5..c303fb8 100644 --- a/GameModel.py +++ b/GameModel.py @@ -1,3 +1,4 @@ +import copy from enum import Enum from typing import List @@ -14,9 +15,12 @@ from PictureVisualizationAgent import PictureVisualizationAgent from data.GameConstants import GameConstants from data.Item import Item from data.Order import Order +from data.enum.ItemType import ItemType from decision.Action import Action from decision.ActionType import ActionType +from imageClasification.Classificator import Classificator from pathfinding.PathfinderOnStates import PathFinderOnStates, PathFinderState +from util.PathByEnum import PathByEnum from util.PathDefinitions import GridLocation, GridWithWeights @@ -30,7 +34,7 @@ class Phase(Enum): class GameModel(Model): - def __init__(self, width, height, graph: GridWithWeights): + def __init__(self, width, height, graph: GridWithWeights, items: int, orders: int, classificator): # self.num_agents = 5 self.first = True self.item_recognised = False @@ -67,14 +71,16 @@ class GameModel(Model): print("############## INITIALIZATION ##############") self.phase = Phase.INIT self.initialize_grid(graph) - self.orderList: List[Order] = InitialStateFactory.generate_order_list(3) + self.orderList: List[Order] = InitialStateFactory.generate_order_list(orders) self.fulfilled_orders: List[Order] = [] self.forklift_agent.orderList = self.orderList self.forklift_agent.fulfilled_orders = self.fulfilled_orders + self.classificator = classificator print("############## RECOGNISE ITEMS ##############") self.phase = Phase.ITEM_RECOGNITION - self.provided_items = InitialStateFactory.generate_item_list(3) + self.provided_items = InitialStateFactory.generate_item_list(items) + self.items_for_recognization = copy.deepcopy(self.provided_items) self.recognised_items: List[Item] = [] print("Relocate forklift agent to loading area for item recognition") @@ -115,7 +121,7 @@ class GameModel(Model): def place_dividers(self): for i in range(0, 10): - for j in range(10,13): + for j in range(10, 13): agent = PatchAgent(self, (i, j), PatchType.divider) self.agents.append(agent) self.grid.place_agent(agent, (i, j)) @@ -160,14 +166,15 @@ class GameModel(Model): if self.phase == Phase.ITEM_RECOGNITION: if not self.item_recognised and self.forklift_agent.current_position == self.drop_off.location: - if len(self.provided_items) == 0: + if len(self.items_for_recognization) == 0: print("FINISHED ITEM RECOGNITION") self.item_recognised = True self.phase = Phase.CLIENT_SORTING + self.forklift_agent.ready_for_execution = True else: - print("BEGIN ITEM RECOGNITION, left: {}".format(len(self.provided_items))) - item_to_recognise = self.provided_items.pop() - self.picture_visualization.img = "item_images/door/drzwi1.jpg" + print("BEGIN ITEM RECOGNITION, left: {}".format(len(self.items_for_recognization))) + item_to_recognise = self.items_for_recognization.pop() + self.picture_visualization.img = PathByEnum.get_random_path(item_to_recognise.real_type) recognised = self.recognise_item(item_to_recognise) self.recognised_items.append(recognised) @@ -182,5 +189,14 @@ class GameModel(Model): def recognise_item(self, item: Item): # TODO IMAGE PROCESSING - item.guessed_type = item.real_type + val = self.classificator.image_clasification(self.picture_visualization.img) + print("VAL: {}".format(val)) + + if val == ItemType.DOOR: + item.guessed_type = ItemType.DOOR + elif val == ItemType.FRIDGE: + item.guessed_type = ItemType.FRIDGE + elif val == ItemType.SHELF: + item.guessed_type = ItemType.SHELF + return item diff --git a/data/enum/ItemType.py b/data/enum/ItemType.py index f15e785..54fc4c1 100644 --- a/data/enum/ItemType.py +++ b/data/enum/ItemType.py @@ -2,6 +2,6 @@ from enum import Enum class ItemType(Enum): - DOOR = 1 - SHELF = 2 - EGG = 3 \ No newline at end of file + DOOR = "door" + SHELF = "shelf" + FRIDGE = "fridge" \ No newline at end of file diff --git a/imageClasification/imageClasification.py b/imageClasification/Classificator.py similarity index 88% rename from imageClasification/imageClasification.py rename to imageClasification/Classificator.py index b8e19b7..4f9d2fd 100644 --- a/imageClasification/imageClasification.py +++ b/imageClasification/Classificator.py @@ -9,8 +9,13 @@ from tensorflow.keras import layers from tensorflow.keras.models import Sequential -class imageClasification(): - def train_model(data_dir): +class Classificator(): + + def __init__(self, data_dir: str) -> None: + super().__init__() + self.data = self.train_model(data_dir) + + def train_model(self, data_dir): batch_size = 32 img_height = 180 img_width = 180 @@ -111,7 +116,7 @@ class imageClasification(): plt.show() return model, class_names, img_width, img_height - def image_clasification(image_path, model, class_names): + def image_clasification(self,image_path): # ścieżka do sprawdzanego obrazu # image_path = "./th-367101945.jpg" @@ -121,11 +126,11 @@ class imageClasification(): img_array = tf.keras.utils.img_to_array(img) img_array = tf.expand_dims(img_array, 0) # Create a batch - predictions = model.predict(img_array) + predictions = self.data[0].predict(img_array) score = tf.nn.softmax(predictions[0]) - # print( - # "This image most likely belongs to {} with a {:.2f} percent confidence." - # .format(class_names[np.argmax(score)], 100 * np.max(score)) - # ) - return class_names[np.argmax(score)] + print( + "This image most likely belongs to {} with a {:.2f} percent confidence." + .format(self.data[1][np.argmax(score)], 100 * np.max(score)) + ) + return self.data[1][np.argmax(score)] diff --git a/imageClasification/how_to_use.py b/imageClasification/how_to_use.py index 225207e..e5e6421 100644 --- a/imageClasification/how_to_use.py +++ b/imageClasification/how_to_use.py @@ -1,15 +1,14 @@ -from imageClasification import imageClasification +from Classificator import Classificator #Training model -data = imageClasification.train_model("../SI_InteligentnyWozekWidlowy/imageClasification/images") - +data = Classificator.train_model("../SI_InteligentnyWozekWidlowy/imageClasification/images") #image_path = ścieżka do klasyfikowanego obrazka #data[0] = model (wytrenowany model #data[1] = class_names -image_classified = imageClasification.image_clasification("./th-367101945.jpg", - data[0], - data[1]) +image_classified = Classificator.image_clasification("./th-367101945.jpg", + data[0], + data[1]) print(image_classified) diff --git a/main.py b/main.py index 0574917..1991f58 100644 --- a/main.py +++ b/main.py @@ -9,11 +9,11 @@ from PatchAgent import PatchAgent from PatchType import PatchType from PictureVisualizationAgent import PictureVisualizationAgent from data.enum.Direction import Direction +from imageClasification.Classificator import Classificator from util.PathDefinitions import GridWithWeights from visualization.DisplayAttributeElement import DisplayAttributeElement from visualization.DisplayItemListAttribute import DisplayItemListAttributeElement from visualization.DisplayOrderList import DisplayOrderList -from visualization.DisplayPictureElement import DisplayPictureElement colors = [ 'blue', 'cyan', 'orange', 'yellow', 'magenta', 'purple', '#103d3e', '#9fc86c', @@ -88,11 +88,13 @@ if __name__ == '__main__': ordersText = DisplayOrderList("orderList") fulfilled_orders = DisplayOrderList("fulfilled_orders") + classificator = Classificator("imageClasification/images") + server = ModularServer(GameModel, [grid, readyText, provided_itesm, recognised_items, ordersText, fulfilled_orders], "Automatyczny Wózek Widłowy", - {"width": gridHeight, "height": gridWidth, "graph": diagram}, ) + dict(width=gridHeight, height=gridWidth, graph=diagram, items=50, orders=3, classificator=classificator)) server.port = 8888 server.launch() diff --git a/util/PathByEnum.py b/util/PathByEnum.py index efe2cee..b517494 100644 --- a/util/PathByEnum.py +++ b/util/PathByEnum.py @@ -1,14 +1,15 @@ -from random import random +import random from data.enum.ItemType import ItemType class PathByEnum: - def get_random_path(self, item:ItemType): + @staticmethod + def get_random_path(item: ItemType): if item == ItemType.DOOR: a = str(random.randint(1, 10)) return "item_images/door/drzwi" + a + ".jpg" - if item == ItemType.EGG: + if item == ItemType.FRIDGE: a = str(random.randint(1, 10)) return "item_images/refrigerator/lodowka" + a + ".jpg" if item == ItemType.SHELF: diff --git a/visualization/DisplayItemListAttribute.py b/visualization/DisplayItemListAttribute.py index 82b2029..9ca1f61 100644 --- a/visualization/DisplayItemListAttribute.py +++ b/visualization/DisplayItemListAttribute.py @@ -41,7 +41,7 @@ class DisplayItemListAttributeElement(TextElement): elif key == ItemType.SHELF: key_str = "Shelf" - res += "
  • {}:{}
    • ".format(key_str, val) + res += "
  • {}:{}
    • ".format(key, val) res += "" diff --git a/visualization/DisplayOrderList.py b/visualization/DisplayOrderList.py index 9b06d35..f7b7564 100644 --- a/visualization/DisplayOrderList.py +++ b/visualization/DisplayOrderList.py @@ -38,13 +38,13 @@ class DisplayOrderList(TextElement): key = e val = itemCounter[key] - key_str = "" - if key == ItemType.DOOR: - key_str = "Door" - elif key == ItemType.SHELF: - key_str = "Shelf" + # key_str = "" + # if key == ItemType.DOOR: + # key_str = "Door" + # elif key == ItemType.SHELF: + # key_str = "Shelf" - item_str += f"
  • {key_str}:{val}
    • " + item_str += f"
  • {str(key)}:{str(val)}
    • " item_str += ""