agent - item recognition loop

ForkliftAgent.py

@@ -32,6 +32,7 @@ class ForkliftAgent(AgentBase):
         self.current_item = None
         self.item_station_completed = False
         self.provided_items: List[Item] = []
+        self.ready_for_execution = False
 
     def queue_movement_actions(self, movement_actions: List[Action]):
         self.action_queue.extend(movement_actions)
@@ -102,7 +103,7 @@ class ForkliftAgent(AgentBase):
                 stations = dict(self.graph.packingStations)
                 if i.real_type == ItemType.SHELF:
                     packing_station = stations[PatchType.packingA]
-                elif i.real_type == ItemType.EGG:
+                elif i.real_type == ItemType.FRIDGE:
                     packing_station = stations[PatchType.packingB]
                 elif i.real_type == ItemType.DOOR:
                     packing_station = stations[PatchType.packingC]
@@ -147,7 +148,7 @@ class ForkliftAgent(AgentBase):
     def step(self) -> None:
         if len(self.action_queue) > 0:
             self.move()
-        elif len(self.orderList) > 0:
+        elif self.ready_for_execution and len(self.orderList) > 0:
             if (self.current_order is not None and len(self.current_order.items)) == 0:
                 self.fulfilled_orders.append(self.current_order)
                 self.current_order = None

GameModel.py

@@ -1,3 +1,4 @@
+import copy
 from enum import Enum
 from typing import List
 
@@ -14,9 +15,12 @@ from PictureVisualizationAgent import PictureVisualizationAgent
 from data.GameConstants import GameConstants
 from data.Item import Item
 from data.Order import Order
+from data.enum.ItemType import ItemType
 from decision.Action import Action
 from decision.ActionType import ActionType
+from imageClasification.Classificator import Classificator
 from pathfinding.PathfinderOnStates import PathFinderOnStates, PathFinderState
+from util.PathByEnum import PathByEnum
 from util.PathDefinitions import GridLocation, GridWithWeights
 
 
@@ -30,7 +34,7 @@ class Phase(Enum):
 
 class GameModel(Model):
 
-    def __init__(self, width, height, graph: GridWithWeights):
+    def __init__(self, width, height, graph: GridWithWeights, items: int, orders: int, classificator):
         # self.num_agents = 5
         self.first = True
         self.item_recognised = False
@@ -67,14 +71,16 @@ class GameModel(Model):
         print("############## INITIALIZATION ##############")
         self.phase = Phase.INIT
         self.initialize_grid(graph)
-        self.orderList: List[Order] = InitialStateFactory.generate_order_list(3)
+        self.orderList: List[Order] = InitialStateFactory.generate_order_list(orders)
         self.fulfilled_orders: List[Order] = []
         self.forklift_agent.orderList = self.orderList
         self.forklift_agent.fulfilled_orders = self.fulfilled_orders
+        self.classificator = classificator
 
         print("############## RECOGNISE ITEMS ##############")
         self.phase = Phase.ITEM_RECOGNITION
-        self.provided_items = InitialStateFactory.generate_item_list(3)
+        self.provided_items = InitialStateFactory.generate_item_list(items)
+        self.items_for_recognization = copy.deepcopy(self.provided_items)
         self.recognised_items: List[Item] = []
 
         print("Relocate forklift agent to loading area for item recognition")
@@ -115,7 +121,7 @@ class GameModel(Model):
 
     def place_dividers(self):
         for i in range(0, 10):
-            for j in range(10,13):
+            for j in range(10, 13):
                 agent = PatchAgent(self, (i, j), PatchType.divider)
                 self.agents.append(agent)
                 self.grid.place_agent(agent, (i, j))
@@ -160,14 +166,15 @@ class GameModel(Model):
         if self.phase == Phase.ITEM_RECOGNITION:
             if not self.item_recognised and self.forklift_agent.current_position == self.drop_off.location:
 
-                if len(self.provided_items) == 0:
+                if len(self.items_for_recognization) == 0:
                     print("FINISHED ITEM RECOGNITION")
                     self.item_recognised = True
                     self.phase = Phase.CLIENT_SORTING
+                    self.forklift_agent.ready_for_execution = True
                 else:
-                    print("BEGIN ITEM RECOGNITION, left: {}".format(len(self.provided_items)))
+                    print("BEGIN ITEM RECOGNITION, left: {}".format(len(self.items_for_recognization)))
-                    item_to_recognise = self.provided_items.pop()
+                    item_to_recognise = self.items_for_recognization.pop()
-                    self.picture_visualization.img = "item_images/door/drzwi1.jpg"
+                    self.picture_visualization.img = PathByEnum.get_random_path(item_to_recognise.real_type)
                     recognised = self.recognise_item(item_to_recognise)
                     self.recognised_items.append(recognised)
 
@@ -182,5 +189,14 @@ class GameModel(Model):
 
     def recognise_item(self, item: Item):
         # TODO IMAGE PROCESSING
-        item.guessed_type = item.real_type
+        val = self.classificator.image_clasification(self.picture_visualization.img)
+        print("VAL: {}".format(val))
+
+        if val == ItemType.DOOR:
+            item.guessed_type = ItemType.DOOR
+        elif val == ItemType.FRIDGE:
+            item.guessed_type = ItemType.FRIDGE
+        elif val == ItemType.SHELF:
+            item.guessed_type = ItemType.SHELF
+
         return item

data/enum/ItemType.py

@@ -2,6 +2,6 @@ from enum import Enum
 
 
 class ItemType(Enum):
-    DOOR = 1
+    DOOR = "door"
-    SHELF = 2
+    SHELF = "shelf"
-    EGG = 3
+    FRIDGE = "fridge"

imageClasification/Classificator.py

@@ -9,8 +9,13 @@ from tensorflow.keras import layers
 from tensorflow.keras.models import Sequential
 
 
-class imageClasification():
+class Classificator():
-    def train_model(data_dir):
+
+    def __init__(self, data_dir: str) -> None:
+        super().__init__()
+        self.data = self.train_model(data_dir)
+
+    def train_model(self, data_dir):
         batch_size = 32
         img_height = 180
         img_width = 180
@@ -111,7 +116,7 @@ class imageClasification():
         plt.show()
         return model, class_names, img_width, img_height
 
-    def image_clasification(image_path, model, class_names):
+    def image_clasification(self,image_path):
         # ścieżka do sprawdzanego obrazu
         # image_path = "./th-367101945.jpg"
 
@@ -121,11 +126,11 @@ class imageClasification():
         img_array = tf.keras.utils.img_to_array(img)
         img_array = tf.expand_dims(img_array, 0)  # Create a batch
 
-        predictions = model.predict(img_array)
+        predictions = self.data[0].predict(img_array)
         score = tf.nn.softmax(predictions[0])
 
-        # print(
+        print(
-        #     "This image most likely belongs to {} with a {:.2f} percent confidence."
+            "This image most likely belongs to {} with a {:.2f} percent confidence."
-        #         .format(class_names[np.argmax(score)], 100 * np.max(score))
+                .format(self.data[1][np.argmax(score)], 100 * np.max(score))
-        # )
+        )
-        return class_names[np.argmax(score)]
+        return self.data[1][np.argmax(score)]

imageClasification/how_to_use.py

@@ -1,15 +1,14 @@
-from imageClasification import imageClasification
+from Classificator import Classificator
 
 #Training model
-data = imageClasification.train_model("../SI_InteligentnyWozekWidlowy/imageClasification/images")
+data = Classificator.train_model("../SI_InteligentnyWozekWidlowy/imageClasification/images")
-
 
 #image_path = ścieżka do klasyfikowanego obrazka
 #data[0] = model (wytrenowany model
 #data[1] = class_names
-image_classified = imageClasification.image_clasification("./th-367101945.jpg",
+image_classified = Classificator.image_clasification("./th-367101945.jpg",
-                                                          data[0],
+                                                     data[0],
-                                                          data[1])
+                                                     data[1])
 
 
 print(image_classified)

main.py

@@ -9,11 +9,11 @@ from PatchAgent import PatchAgent
 from PatchType import PatchType
 from PictureVisualizationAgent import PictureVisualizationAgent
 from data.enum.Direction import Direction
+from imageClasification.Classificator import Classificator
 from util.PathDefinitions import GridWithWeights
 from visualization.DisplayAttributeElement import DisplayAttributeElement
 from visualization.DisplayItemListAttribute import DisplayItemListAttributeElement
 from visualization.DisplayOrderList import DisplayOrderList
-from visualization.DisplayPictureElement import DisplayPictureElement
 
 colors = [
     'blue', 'cyan', 'orange', 'yellow', 'magenta', 'purple', '#103d3e', '#9fc86c',
@@ -88,11 +88,13 @@ if __name__ == '__main__':
     ordersText = DisplayOrderList("orderList")
     fulfilled_orders = DisplayOrderList("fulfilled_orders")
 
+    classificator = Classificator("imageClasification/images")
+
     server = ModularServer(GameModel,
                            [grid, readyText, provided_itesm, recognised_items, ordersText,
                             fulfilled_orders],
                            "Automatyczny Wózek Widłowy",
-                           {"width": gridHeight, "height": gridWidth, "graph": diagram}, )
+                           dict(width=gridHeight, height=gridWidth, graph=diagram, items=50, orders=3, classificator=classificator))
 
     server.port = 8888
     server.launch()

util/PathByEnum.py

@@ -1,14 +1,15 @@
-from random import random
+import random
 
 from data.enum.ItemType import ItemType
 
 
 class PathByEnum:
-    def get_random_path(self, item:ItemType):
+    @staticmethod
+    def get_random_path(item: ItemType):
         if item == ItemType.DOOR:
             a = str(random.randint(1, 10))
             return "item_images/door/drzwi" + a + ".jpg"
-        if item == ItemType.EGG:
+        if item == ItemType.FRIDGE:
             a = str(random.randint(1, 10))
             return "item_images/refrigerator/lodowka" + a + ".jpg"
         if item == ItemType.SHELF:

visualization/DisplayItemListAttribute.py

@@ -41,7 +41,7 @@ class DisplayItemListAttributeElement(TextElement):
             elif key == ItemType.SHELF:
                 key_str = "Shelf"
 
-            res += "<li>{}:{}</li>".format(key_str, val)
+            res += "<li>{}:{}</li>".format(key, val)
 
         res += "</ul>"
 

visualization/DisplayOrderList.py

@@ -38,13 +38,13 @@ class DisplayOrderList(TextElement):
                 key = e
                 val = itemCounter[key]
 
-                key_str = ""
+                # key_str = ""
-                if key == ItemType.DOOR:
+                # if key == ItemType.DOOR:
-                    key_str = "Door"
+                #     key_str = "Door"
-                elif key == ItemType.SHELF:
+                # elif key == ItemType.SHELF:
-                    key_str = "Shelf"
+                #     key_str = "Shelf"
 
-                item_str += f"<li>{key_str}:{val}</li>"
+                item_str += f"<li>{str(key)}:{str(val)}</li>"
 
             item_str += "</ul>"