Merge pull request 'development' (#12) from development into master

Reviewed-on: #12

data.txt

@@ -0,0 +1,100 @@
+{"weight": 9, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 11, "eatable": true, "toughness": 0, "resource": "wood"}
+{"weight": 5, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 11, "eatable": true, "toughness": 0, "resource": "wood"}
+{"weight": 7, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 7, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 5, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 5, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 1, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 2, "eatable": false, "toughness": 0, "resource": "wood"}
+{"weight": 4, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 1, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 3, "eatable": false, "toughness": 2, "resource": "wood"}
+{"weight": 6, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 7, "eatable": false, "toughness": 0, "resource": "wood"}
+{"weight": 1, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 6, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 2, "eatable": false, "toughness": 2, "resource": "wood"}
+{"weight": 10, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 6, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 11, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 2, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 3, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 6, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 8, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 6, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 9, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 9, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 11, "eatable": false, "toughness": 0, "resource": "wood"}
+{"weight": 4, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 4, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 4, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 1, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 5, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 3, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 0, "eatable": false, "toughness": 2, "resource": "wood"}
+{"weight": 10, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 9, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 4, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 10, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 10, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 11, "eatable": false, "toughness": 0, "resource": "wood"}
+{"weight": 8, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 7, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 1, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 11, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 11, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 3, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 2, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 4, "eatable": false, "toughness": 2, "resource": "wood"}
+{"weight": 6, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 2, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 10, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 0, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 10, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 9, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 2, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 8, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 9, "eatable": false, "toughness": 2, "resource": "wood"}
+{"weight": 6, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 5, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 4, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 0, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 8, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 2, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 2, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 11, "eatable": false, "toughness": 0, "resource": "wood"}
+{"weight": 0, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 9, "eatable": false, "toughness": 0, "resource": "wood"}
+{"weight": 0, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 5, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 3, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 0, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 7, "eatable": false, "toughness": 2, "resource": "wood"}
+{"weight": 4, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 4, "eatable": false, "toughness": 0, "resource": "wood"}
+{"weight": 3, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 8, "eatable": false, "toughness": 3, "resource": "wood"}
+{"weight": 10, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 3, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 3, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 1, "eatable": false, "toughness": 2, "resource": "wood"}
+{"weight": 1, "eatable": false, "toughness": 2, "resource": "wood"}
+{"weight": 5, "eatable": true, "toughness": 1, "resource": "food"}
+{"weight": 5, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 9, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 3, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 6, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 7, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 9, "eatable": false, "toughness": 0, "resource": "wood"}
+{"weight": 10, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 9, "eatable": false, "toughness": 0, "resource": "wood"}
+{"weight": 11, "eatable": true, "toughness": 0, "resource": "wood"}
+{"weight": 10, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 5, "eatable": true, "toughness": 3, "resource": "food"}
+{"weight": 10, "eatable": false, "toughness": 1, "resource": "wood"}
+{"weight": 10, "eatable": true, "toughness": 2, "resource": "food"}
+{"weight": 5, "eatable": true, "toughness": 0, "resource": "water"}
+{"weight": 0, "eatable": false, "toughness": 2, "resource": "wood"}
+{"weight": 10, "eatable": false, "toughness": 2, "resource": "wood"}

generate_test_data.py

@@ -0,0 +1,28 @@
+import random
+
+
+def generate_data():
+    f = open("data.txt", "w")
+    for i in range(100):
+        weight = random.randint(0, 11)
+        eatable = bool(random.randint(0, 1))
+        toughness = random.randint(0, 3)
+        f.write('{')
+        f.write(
+            f'"weight": {weight}, "eatable": {str(eatable).lower()}, "toughness": {toughness}, "resource": "{get_resource_type(weight, eatable, toughness)}"')
+        f.write('}')
+        f.write('\n')
+    f.close()
+
+
+def get_resource_type(weight, eatable, toughness):
+    if weight > 10 or eatable is False:
+        return "wood"
+
+    if toughness < 1:
+        return "water"
+
+    return "food"
+
+
+generate_data()

survival/__init__.py

@@ -1,14 +1,47 @@
 import pygame
 
-from settings import SCREEN_WIDTH, SCREEN_HEIGHT
-from survival.camera import Camera
+from survival.ai.genetic_algorithm import GeneticAlgorithm
 from survival.components.inventory_component import InventoryComponent
-from survival.game_map import GameMap
+from survival.game.game_map import GameMap
 from survival.generators.building_generator import BuildingGenerator
 from survival.generators.player_generator import PlayerGenerator
 from survival.generators.resource_generator import ResourceGenerator
 from survival.generators.world_generator import WorldGenerator
+from survival.settings import SCREEN_WIDTH, SCREEN_HEIGHT, MUTATE_NETWORKS, LEARN
 from survival.systems.draw_system import DrawSystem
+from survival.systems.neural_system import NeuralSystem
+
+
+class Game:
+    def __init__(self):
+        self.world_generator = WorldGenerator(win, self.reset)
+        self.game_map, self.world, self.camera = self.world_generator.create_world()
+        self.run = True
+        if LEARN and MUTATE_NETWORKS:
+            self.genetic_algorithm = GeneticAlgorithm(self.world.get_processor(NeuralSystem), self.finish_training)
+
+    def reset(self):
+        if LEARN and MUTATE_NETWORKS:
+            self.genetic_algorithm.train()
+        self.world_generator.reset_world()
+
+    def finish_training(self):
+        self.run = False
+
+    def update(self, ms):
+        events = pygame.event.get()
+
+        for event in events:
+            if event.type == pygame.QUIT:
+                self.run = False
+            if pygame.key.get_pressed()[pygame.K_DELETE]:
+                self.reset()
+
+        win.fill((0, 0, 0))
+        self.game_map.draw(self.camera)
+        self.world.process(ms)
+        pygame.display.update()
+
 
 if __name__ == '__main__':
     pygame.init()
@@ -19,32 +52,7 @@ if __name__ == '__main__':
     pygame.display.set_caption("AI Project")
 
     clock = pygame.time.Clock()
+    game = Game()
 
-    game_map = GameMap(int(SCREEN_WIDTH / 32) * 2, 2 * int(SCREEN_HEIGHT / 32) + 1)
-    camera = Camera(game_map.width * 32, game_map.height * 32, win)
-
-    world = WorldGenerator().create_world(camera, game_map)
-    player = PlayerGenerator().create_player(world, game_map)
-    world.get_processor(DrawSystem).initialize_interface(world.component_for_entity(player, InventoryComponent))
-    building = BuildingGenerator().create_home(world, game_map)
-
-    ResourceGenerator(world, game_map).generate_resources(player)
-
-    run = True
-
-    while run:
-        # Set the framerate
-        ms = clock.tick(60)
-
-        events = pygame.event.get()
-
-        for event in events:
-            if event.type == pygame.QUIT:
-                run = False
-
-        keys = pygame.key.get_pressed()
-
-        win.fill((0, 0, 0))
-        game_map.draw(camera)
-        world.process(ms)
-        pygame.display.update()
+    while game.run:
+        game.update(clock.tick(500))

survival/ai/decision_tree/decision_tree.py

@@ -0,0 +1,60 @@
+import json
+
+from joblib import dump, load
+from matplotlib import pyplot as plt
+from sklearn import tree
+from sklearn.feature_extraction import DictVectorizer
+
+
+class DecisionTree:
+    def __init__(self):
+        self.clf = None
+        self.vec = None
+
+    def build(self, depth: int):
+        path = "tree_data.json"
+
+        samples = []
+        results = []
+
+        with open(path, "r") as training_file:
+            for sample in training_file:
+                sample, result = self.process_input(sample)
+                samples.append(sample)
+                results.append(result)
+
+        self.vec = DictVectorizer()
+        self.clf = tree.DecisionTreeClassifier(max_depth=depth)
+        self.clf = self.clf.fit(self.vec.fit_transform(samples).toarray(), results)
+
+    def save_model(self, clf_file, vec_file):
+        dump(self.clf, clf_file)
+        dump(self.vec, vec_file)
+
+    def load_model(self, clf_file, vec_file):
+        self.clf = load(clf_file)
+        self.vec = load(vec_file)
+
+    def predict_answer(self, params):
+        return self.clf.predict(self.vec.transform(params).toarray())
+
+    def plot_tree(self):
+        print('Plotting tree...')
+        fig = plt.figure(figsize=(36, 27))
+        _ = tree.plot_tree(self.clf,
+                           feature_names=self.vec.get_feature_names(),
+                           filled=True)
+        fig.savefig("decistion_tree.png")
+        print('Success!')
+
+    @staticmethod
+    def process_input(line):
+        data = json.loads(line.strip())
+        result = data['result']
+        del data['result']
+        del data['food_result']
+        del data['water_result']
+        del data['wood_result']
+        sample = data
+
+        return sample, result

survival/ai/decision_tree/decision_tree_data.py

@@ -0,0 +1,124 @@
+import random
+from typing import Dict
+
+from survival.ai.decision_tree.decision_tree import DecisionTree
+from survival.generators.resource_type import ResourceType
+
+
+class TreeDataGenerator:
+    INV_RANGE = (1, 100)
+    VISIBLE = (True, False)
+    DISTANCE_RANGE = (3, 7)
+    DISTANCE_FACTOR = 0.2
+    COUNT = (1, 2, 3)
+
+    def generate(self, count=1000):
+        full_data = []
+        self.process(count, full_data)
+
+        self.write_data_to_file(full_data)
+        return full_data
+
+    def process(self, count, full_data):
+        for i in range(count):
+            # if i % 10000 == 0:
+            #     print(i)
+            package = {}
+
+            # Create resource data for each resource type.
+            for resource in ResourceType:
+                package[resource] = self.create_resource_data()
+
+            # Get the resource with highest result among all generated resource types.
+            best_resource = self.get_best_resource(package)
+
+            # Unpack packaged resources.
+            (food, water, wood) = (
+                package[ResourceType.FOOD], package[ResourceType.WATER], package[ResourceType.WOOD])
+
+            # Create dictionary filled with data.
+            data = {"food_inv": food[0], 'food_visible': str(food[1]), 'food_distance': food[2],
+                    'food_count': food[3], 'food_result': food[4],
+                    'water_inv': water[0], 'water_visible': str(water[1]), 'water_distance': water[2],
+                    'water_count': water[3], 'water_result': water[4],
+                    'wood_inv': wood[0], 'wood_visible': str(wood[1]), 'wood_distance': wood[2],
+                    'wood_count': wood[3], 'wood_result': wood[4],
+                    'result': best_resource.name.lower()}
+            full_data.append(data)
+
+    @staticmethod
+    def write_data_to_file(full_data):
+        print("Writing to file...")
+        # Open the target file to which the data will be saved and write all the data to it.
+        with open('tree_data.json', 'w') as f:
+            for data in full_data:
+                data_str = str(data).replace("'", '"').replace('"False"', 'false').replace('"True"', 'true')
+                f.write(data_str)
+                f.write('\n')
+        print("Success!")
+
+    def create_resource_data(self):
+        is_visible = random.choice(self.VISIBLE)
+        inventory = random.randint(min(self.INV_RANGE), max(self.INV_RANGE))
+
+        if is_visible:
+            cnt = random.choice(self.COUNT)
+            distance = random.randint(min(self.DISTANCE_RANGE), max(self.DISTANCE_RANGE))
+        else:
+            cnt = 0
+            distance = 0
+
+        # Equation determining the results processed by decision tree.
+        result = (self.INV_RANGE[1] / inventory) * (1 * cnt if is_visible else 0.9) + (
+            max(self.DISTANCE_RANGE) / distance if is_visible else 0.5) * self.DISTANCE_FACTOR
+
+        return [inventory, is_visible, distance, cnt, result]
+
+    @staticmethod
+    def get_best_resource(package: Dict) -> ResourceType:
+        best_resource = None
+        for resource, data in package.items():
+            if best_resource is None or data[:-1] < package[best_resource][:-1]:
+                best_resource = resource
+        return best_resource
+
+    @staticmethod
+    def print_data(full_data):
+        for data in full_data:
+            print(TreeDataGenerator.format_words(["Data", "Apple", "Water", "Wood"]))
+            print(TreeDataGenerator.format_words(["Inventory", data["food_inv"], data["water_inv"], data["wood_inv"]]))
+            print(TreeDataGenerator.format_words(
+                ["Visible", data["food_visible"], data["water_visible"], data["wood_visible"]]))
+            print(TreeDataGenerator.format_words(
+                ["Distance", data["food_distance"], data["water_distance"], data["wood_distance"]]))
+            print(
+                TreeDataGenerator.format_words(["Count", data["food_count"], data["water_count"], data["wood_count"]]))
+            print(TreeDataGenerator.format_words(
+                ["Result", round(data["food_result"], 3), round(data["water_result"], 3),
+                 round(data["wood_result"], 3)]))
+            print(f'Best resource: {data["result"]}')
+            print('--------------------------------------------------------------')
+
+    @staticmethod
+    def format_words(words):
+        return '{:>12}  {:>12}  {:>12} {:>12}'.format(words[0], words[1], words[2], words[3])
+
+
+# Train tree
+generator = TreeDataGenerator()
+data = generator.generate(50000)
+generator.print_data(data)
+tree = DecisionTree()
+tree.build(1000)
+tree.plot_tree()
+tree.save_model('classifier.joblib', 'vectorizer.joblib')
+# ----------------------------------------------------------- #
+# Use trained tree
+# tree = DecisionTree()
+# tree.load_model('classifier.joblib', 'vectorizer.joblib')
+#
+# answ = tree.predict_answer({'food_inv': 40, 'water_inv': 10, 'wood_inv': 20,
+#                      'food_distance': 2, 'water_distance': -1, 'wood_distance': 4,
+#                      'food_visible': True, 'water_visible': False, 'wood_visible': True,
+#                      'food_count': 1, 'water_count': 1, 'wood_count': 1})
+# print(answ)

survival/ai/genetic_algorithm.py

@@ -0,0 +1,84 @@
+import sys
+
+from survival.ai.model import LinearQNetwork
+from survival.ai.optimizer import Optimizer
+
+
+class GeneticAlgorithm:
+    GAMES_PER_NETWORK = 40
+    PLOTS_COUNTER = 0
+    CURRENT_GENERATION = 1
+
+    def __init__(self, neural_system, callback):
+        self.callback = callback
+        self.logs_file = open('genetic_logs.txt', 'w')
+        self.original_stdout = sys.stdout
+        sys.stdout = self.logs_file
+        self.neural_system = neural_system
+        self.generations = 20
+        self.population = 10  # Minimum 5 needed to allow breeding
+        self.nn_params = {
+            'neurons': [128, 192, 256, 384, 512],
+            'layers': [0, 1, 2, 3],
+            'activation': ['relu', 'elu', 'tanh'],
+            'ratio': [0.0007, 0.0009, 0.0011, 0.0013, 0.0015],
+            'optimizer': ['RMSprop', 'Adam', 'SGD', 'Adagrad', 'Adadelta'],
+        }
+        self.optimizer = Optimizer(self.nn_params)
+        self.networks: list[LinearQNetwork] = self.optimizer.create_population(self.population)
+        self.finished = False
+        self.trained_counter = 0
+        self.iterations = 0
+        self.trained_generations = 0
+        print('Started generation 1...')
+        self.change_network(self.networks[0])
+
+    def train(self):
+        if self.iterations < GeneticAlgorithm.GAMES_PER_NETWORK - 1:
+            self.iterations += 1
+            return
+        self.iterations = 0
+        print(f'Network score: {self.optimizer.fitness(self.networks[self.trained_counter])}')
+        self.trained_counter += 1
+
+        # If all networks in current population were trained
+        if self.trained_counter >= self.population:
+            # Get average score in current population
+            avg_score = self.calculate_average_score(self.networks)
+            print(f'Average population score: {avg_score}.')
+
+            results_file = open('genetic_results.txt', 'w')
+            for network in self.networks:
+                results_file.write(
+                    f'Network {network.id} params {network.network_params}. Avg score = {sum(network.scores) / len(network.scores)}\n')
+            results_file.close()
+            if self.trained_generations >= self.generations - 1:
+                # Sort the final population
+                self.networks = sorted(self.networks, key=lambda x: sum(x.scores) / len(x.scores), reverse=True)
+                self.finished = True
+                self.logs_file.close()
+                sys.stdout = self.original_stdout
+                self.callback()
+                return
+
+            self.trained_generations += 1
+            GeneticAlgorithm.CURRENT_GENERATION = self.trained_generations + 1
+            print(f'Started generation {GeneticAlgorithm.CURRENT_GENERATION}...')
+            self.networks = self.optimizer.evolve(self.networks)
+            self.trained_counter = 0
+        self.change_network(self.networks[self.trained_counter])
+
+    def calculate_average_score(self, networks):
+        sums = 0
+        lengths = 0
+        for network in networks:
+            sums += self.optimizer.fitness(network)
+            lengths += len(network.scores)
+        return sums / lengths
+
+    def change_network(self, net):
+        GeneticAlgorithm.PLOTS_COUNTER += 1
+        print(f"Changed network to {GeneticAlgorithm.PLOTS_COUNTER} {net.network_params}")
+        self.logs_file.flush()
+        net.id = GeneticAlgorithm.PLOTS_COUNTER
+        self.neural_system.load_model(net)

survival/ai/graph_search.py

@@ -2,9 +2,13 @@ from enum import Enum
 from queue import PriorityQueue
 from typing import Tuple, List
 
-from survival import GameMap
+from survival.components.direction_component import DirectionChangeComponent
+from survival.components.moving_component import MovingComponent
 from survival.components.position_component import PositionComponent
-from survival.enums import Direction
+from survival.components.resource_component import ResourceComponent
+from survival.game.enums import Direction
+from survival.esper import World
+from survival.systems.consumption_system import ConsumeComponent
 
 
 class Action(Enum):
@@ -12,6 +16,35 @@ class Action(Enum):
     ROTATE_RIGHT = 1
     MOVE = 2
 
+    @staticmethod
+    def from_array(action):
+        if action[0] == 1:
+            return Action.MOVE
+        if action[1] == 1:
+            return Action.ROTATE_LEFT
+        if action[2] == 1:
+            return Action.ROTATE_RIGHT
+        raise Exception("Unknown action.")
+
+    @staticmethod
+    def perform(world, entity, action):
+        if world.has_component(entity, MovingComponent):
+            raise Exception(f"Entity was already moving. Could not perform action: {action}")
+        if world.has_component(entity, DirectionChangeComponent):
+            raise Exception(f"Entity was already rotating. Could not perform action: {action}")
+
+        if action == Action.ROTATE_LEFT:
+            world.add_component(entity, DirectionChangeComponent(
+                Direction.rotate_left(world.component_for_entity(entity, PositionComponent).direction)))
+            world.add_component(entity, ConsumeComponent(0.2))
+        elif action == Action.ROTATE_RIGHT:
+            world.add_component(entity, DirectionChangeComponent(
+                Direction.rotate_right(world.component_for_entity(entity, PositionComponent).direction)))
+            world.add_component(entity, ConsumeComponent(0.2))
+        else:
+            world.add_component(entity, MovingComponent())
+        return action
+
 
 class State:
     def __init__(self, position: Tuple[int, int], direction: Direction):
@@ -38,7 +71,7 @@ def get_moved_position(position: Tuple[int, int], direction: Direction):
     return position[0] + vector[0], position[1] + vector[1]
 
 
-def get_states(state: State, game_map: GameMap) -> List[Tuple[Action, State, int]]:
+def get_states(state: State, game_map, world: World) -> List[Tuple[Action, State, int]]:
     states = list()
 
     states.append((Action.ROTATE_LEFT, State(state.position, state.direction.rotate_left(state.direction)), 1))
@@ -47,28 +80,34 @@ def get_states(state: State, game_map: GameMap) -> List[Tuple[Action, State, int
     target_position = get_moved_position(state.position, state.direction)
     if not game_map.is_colliding(target_position):
         states.append((Action.MOVE, State(target_position, state.direction), game_map.get_cost(target_position)))
+    elif game_map.get_entity(target_position) is not None:
+        ent = game_map.get_entity(target_position)
+        if world.has_component(ent, ResourceComponent):
+            states.append((Action.MOVE, State(target_position, state.direction), 3))
 
     return states
 
 
 def build_path(node: Node):
+    cost = 0
     actions = [node.action]
     parent = node.parent
 
     while parent is not None:
         if parent.action is not None:
             actions.append(parent.action)
+            cost += parent.cost
         parent = parent.parent
 
     actions.reverse()
-    return actions
+    return actions, cost
 
 
 def heuristic(new_node: Node, goal: Tuple[int, int]):
     return abs(new_node.state.position[0] - goal[0]) + abs(new_node.state.position[1] - goal[1])
 
 
-def graph_search(game_map: GameMap, start: PositionComponent, goal: tuple):
+def graph_search(game_map, start: PositionComponent, goal: tuple, world: World):
     fringe = PriorityQueue()
     explored = list()
 
@@ -82,7 +121,7 @@ def graph_search(game_map: GameMap, start: PositionComponent, goal: tuple):
     while True:
         # No solutions found
         if fringe.empty():
-            return []
+            return [], 0
 
         node = fringe.get()
         node_priority = node[0]
@@ -97,13 +136,13 @@ def graph_search(game_map: GameMap, start: PositionComponent, goal: tuple):
         explored_states.add((tuple(node.state.position), node.state.direction))
 
         # Get all possible states
-        for state in get_states(node.state, game_map):
+        for state in get_states(node.state, game_map, world):
             sub_state = (tuple(state[1].position), state[1].direction)
             new_node = Node(state=state[1],
                             parent=node,
                             action=state[0],
                             cost=(state[2] + node.cost))
-            
+
             priority = new_node.cost + heuristic(new_node, goal)
             if sub_state not in fringe_states and sub_state not in explored_states:
                 fringe.put((priority, new_node))

survival/ai/learning_utils.py

@@ -0,0 +1,120 @@
+import numpy as np
+from IPython import display
+from matplotlib import pyplot as plt
+
+from survival.settings import MUTATE_NETWORKS
+from survival.ai.genetic_algorithm import GeneticAlgorithm
+from survival.components.learning_component import LearningComponent
+from survival.components.position_component import PositionComponent
+from survival.game.enums import Direction
+from survival.ai.graph_search import Action
+
+
+class LearningUtils:
+    def __init__(self):
+        self.plot_scores = []
+        self.plot_mean_scores = []
+        self.total_score = 0
+        self.last_actions: [Action, [int, int]] = []
+        self.plots = 0
+
+    def add_scores(self, learning: LearningComponent, games_count: int):
+        self.plot_scores.append(learning.score)
+        self.total_score += learning.score
+        mean_score = self.total_score / games_count
+        self.plot_mean_scores.append(mean_score)
+
+    def plot(self):
+        # display.clear_output(wait=True)
+        # display.display(plt.gcf())
+        plt.clf()
+        plt.title('Results')
+        plt.xlabel('Number of Games')
+        plt.ylabel('Score')
+        plt.plot(self.plot_scores)
+        plt.plot(self.plot_mean_scores)
+        plt.ylim(ymin=0)
+        plt.text(len(self.plot_scores) - 1, self.plot_scores[-1], str(self.plot_scores[-1]))
+        plt.text(len(self.plot_mean_scores) - 1, self.plot_mean_scores[-1], str(self.plot_mean_scores[-1]))
+        self.plots += 1
+        if MUTATE_NETWORKS:
+            plt.savefig(f'model/plots/{GeneticAlgorithm.PLOTS_COUNTER}_{self.plots}.png')
+        else:
+            plt.savefig(f'model/plots/{self.plots}.png')
+        plt.show(block=False)
+        # plt.pause(.1)
+
+    def append_action(self, action: Action, pos: PositionComponent):
+        self.last_actions.append([action, pos.grid_position])
+
+    def check_last_actions(self, learning):
+        """
+        Checks if all the last five actions were repeated and imposes the potential penalty.
+        :param learning:
+        """
+        if len(self.last_actions) > 5:
+            self.last_actions.pop(0)
+
+        last_action: [Action, [int, int]] = self.last_actions[0]
+        last_grid_pos: [int, int] = last_action[1]
+
+        rotations = 0
+        collisions = 0
+        for action in self.last_actions:
+            if action != Action.MOVE:
+                rotations += 1
+            else:
+                current_grid_pos = action[1]
+                if current_grid_pos[0] == last_grid_pos[0] and current_grid_pos[1] == last_grid_pos[1]:
+                    collisions += 1
+
+        if rotations > 4 or collisions > 4:
+            learning.reward -= 2
+
+
+def get_state(system, player, resource):
+    pos: PositionComponent = system.world.component_for_entity(player, PositionComponent)
+    if resource is None or resource[0] is None:
+        res_l = False
+        res_r = False
+        res_u = False
+        res_d = False
+    else:
+        resource_pos: PositionComponent = system.world.component_for_entity(resource[0], PositionComponent)
+        res_l = resource_pos.grid_position[0] < pos.grid_position[0]
+        res_r = resource_pos.grid_position[0] > pos.grid_position[0]
+        res_u = resource_pos.grid_position[1] < pos.grid_position[1]
+        res_d = resource_pos.grid_position[1] > pos.grid_position[1]
+
+    dir_l = pos.direction == Direction.LEFT
+    dir_r = pos.direction == Direction.RIGHT
+    dir_u = pos.direction == Direction.UP
+    dir_d = pos.direction == Direction.DOWN
+
+    pos_l = [pos.grid_position[0] - 1, pos.grid_position[1]]
+    pos_r = [pos.grid_position[0] + 1, pos.grid_position[1]]
+    pos_u = [pos.grid_position[0], pos.grid_position[1] - 1]
+    pos_d = [pos.grid_position[0], pos.grid_position[1] + 1]
+    col_l = system.game_map.in_bounds(
+        pos_l)  # self.game_map.is_colliding(pos_l) and self.game_map.get_entity(pos_l) is None
+    col_r = system.game_map.in_bounds(
+        pos_r)  # self.game_map.is_colliding(pos_r) and self.game_map.get_entity(pos_r) is None
+    col_u = system.game_map.in_bounds(
+        pos_u)  # self.game_map.is_colliding(pos_u) and self.game_map.get_entity(pos_u) is None
+    col_d = system.game_map.in_bounds(
+        pos_d)  # self.game_map.is_colliding(pos_d) and self.game_map.get_entity(pos_d) is None
+
+    state = [
+        # Collision ahead
+        (dir_r and col_r) or (dir_l and col_l) or (dir_u and col_u) or (dir_d and col_d),
+        # Collision on the right
+        (dir_u and col_r) or (dir_r and col_d) or (dir_d and col_l) or (dir_l and col_u),
+        # Collision on the left
+        (dir_u and col_l) or (dir_l and col_d) or (dir_d and col_r) or (dir_r and col_u),
+        # Movement direction
+        dir_l, dir_r, dir_u, dir_d,
+        # Resource location
+        res_l, res_r, res_u, res_d
+    ]
+
+    return np.array(state, dtype=int)

survival/ai/model.py

@@ -0,0 +1,110 @@
+import os
+import random
+
+import torch
+from torch import nn, optim
+import torch.nn.functional as functional
+
+
+class LinearQNetwork(nn.Module):
+    TORCH_ACTiVATIONS = 'tanh'
+
+    def __init__(self, nn_params, input_size, output_size, randomize=True, params=None):
+        super().__init__()
+        self.id = 0
+        if params is None:
+            params = {}
+        self.params_choice = nn_params
+        self.scores = []
+        self.network_params = params
+        if randomize:
+            self.randomize()
+        self.layers = nn.ModuleList()
+        if self.network_params['layers'] == 0:
+            self.layers.append(nn.Linear(input_size, output_size))
+        else:
+            self.layers.append(nn.Linear(input_size, self.network_params['neurons']))
+
+        for i in range(self.network_params['layers'] - 1):
+            self.layers.append(nn.Linear(self.network_params['neurons'], self.network_params['neurons']))
+        if self.network_params['layers'] > 0:
+            self.ending_linear = nn.Linear(self.network_params['neurons'], output_size)
+            self.layers.append(self.ending_linear)
+
+        if self.network_params['activation'] in self.TORCH_ACTiVATIONS:
+            self.forward_func = getattr(torch, self.network_params['activation'])
+        else:
+            self.forward_func = getattr(functional, self.network_params['activation'])
+
+    def randomize(self):
+        """
+        Sets random parameters for network.
+        """
+        for key in self.params_choice:
+            self.network_params[key] = random.choice(self.params_choice[key])
+
+    def forward(self, x):
+        for i in range(len(self.layers) - 1):
+            x = self.forward_func(self.layers[i](x))
+        x = self.layers[-1](x)
+        return x
+
+    def save(self, file_name='model.pth'):
+        model_directory = 'model'
+        if not os.path.exists(model_directory):
+            os.makedirs(model_directory)
+
+        file_path = os.path.join(model_directory, file_name)
+        torch.save(self.state_dict(), file_path)
+
+    @staticmethod
+    def load(params, input_size, output_size, file_name='model.pth'):
+        model_directory = 'model'
+        file_path = os.path.join(model_directory, file_name)
+        if os.path.isfile(file_path):
+            model = LinearQNetwork(params, input_size, output_size, True)
+            model.load_state_dict(torch.load(file_path))
+            model.eval()
+            return model
+        raise Exception(f'Could not find file {file_path}.')
+
+
+class QTrainer:
+    def __init__(self, model, lr, gamma, optimizer):
+        self.model = model
+        self.lr = lr
+        self.gamma = gamma
+        self.optimizer = getattr(optim, optimizer)(model.parameters(), lr=self.lr)
+        # self.optimizer = optim.Adam(model.parameters(), lr=self.lr)
+        self.criterion = nn.MSELoss()  # Mean squared error
+
+    def train_step(self, state, action, reward, next_state, done):
+        state = torch.tensor(state, dtype=torch.float)
+        next_state = torch.tensor(next_state, dtype=torch.float)
+        action = torch.tensor(action, dtype=torch.long)
+        reward = torch.tensor(reward, dtype=torch.float)
+
+        if len(state.shape) == 1:
+            # reshape the state to make its values an (n, x) tuple
+            state = torch.unsqueeze(state, 0)
+            next_state = torch.unsqueeze(next_state, 0)
+            action = torch.unsqueeze(action, 0)
+            reward = torch.unsqueeze(reward, 0)
+            done = (done,)
+
+        # Prediction based on simplified Bellman's equation
+        # Predict Q values for current state
+        prediction = self.model(state)
+        target = prediction.clone()
+        for idx in range(len(done)):
+            Q = reward[idx]
+            if not done[idx]:
+                Q = reward[idx] + self.gamma * torch.max(self.model(next_state[idx]))
+            # set the target of the maximum value of the action to Q
+            target[idx][torch.argmax(action).item()] = Q
+        # Apply the loss function
+        self.optimizer.zero_grad()
+        loss = self.criterion(target, prediction)
+        loss.backward()
+
+        self.optimizer.step()

survival/ai/optimizer.py

@@ -0,0 +1,144 @@
+from functools import reduce
+from operator import add
+import random
+from typing import List
+
+from survival.ai.model import LinearQNetwork
+from survival.settings import NEURAL_INPUT_SIZE, NEURAL_OUTPUT_SIZE
+
+
+class Optimizer:
+    def __init__(self, params, retain=0.4,
+                 random_select=0.1, mutation_chance=0.2):
+        self.mutation_chance = mutation_chance
+        self.random_select = random_select
+        self.retain = retain
+        self.nn_params = params
+
+    def create_population(self, count: int):
+        """
+        Creates 'count' networks from random parameters.
+        :param count:
+        :return:
+        """
+        pop = []
+        for _ in range(0, count):
+            # Create a random network.
+            network = LinearQNetwork(self.nn_params, NEURAL_INPUT_SIZE, NEURAL_OUTPUT_SIZE)
+            # Add network to the population.
+            pop.append(network)
+
+        return pop
+
+    @staticmethod
+    def fitness(network: LinearQNetwork):
+        return sum(network.scores) / len(network.scores)
+
+    def grade(self, pop: List[LinearQNetwork]) -> float:
+        """
+        Finds average fitness for given population.
+        """
+        summed = reduce(add, (self.fitness(network) for network in pop))
+        return summed / float((len(pop)))
+
+    def breed(self, parent_one, parent_two):
+        """
+        Creates a new network from given parents.
+        :param parent_one:
+        :param parent_two:
+        :return:
+        """
+        children = []
+        for _ in range(2):
+
+            child = {}
+
+            # Loop through the parameters and pick params for the kid.
+            for param in self.nn_params:
+                child[param] = random.choice(
+                    [parent_one.network_params[param], parent_two.network_params[param]]
+                )
+
+            # Create new network object.
+            network = LinearQNetwork(self.nn_params, NEURAL_INPUT_SIZE, NEURAL_OUTPUT_SIZE)
+            network.network_params = child
+
+            children.append(network)
+
+        return children
+
+    def mutate(self, network: LinearQNetwork):
+        """
+        Randomly mutates one parameter of the given network.
+        :param network:
+        :return:
+        """
+        mutation = random.choice(list(self.nn_params.keys()))
+
+        # Mutate one of the params.
+        network.network_params[mutation] = random.choice(self.nn_params[mutation])
+
+        return network
+
+    def evolve(self, pop):
+        """
+        Evolves a population of networks.
+        """
+        # Get scores for each network.
+        scores = [(self.fitness(network), network) for network in pop]
+
+        # Sort the scores.
+        scores = [x[1] for x in sorted(scores, key=lambda x: x[0], reverse=True)]
+
+        # Get the number we want to keep for the next gen.
+        retain_length = int(len(scores) * self.retain)
+
+        # Keep the best networks as parents for next generation.
+        parents = scores[:retain_length]
+
+        # Keep some other networks
+        for network in scores[retain_length:]:
+            if self.random_select > random.random():
+                parents.append(network)
+
+        # Reset kept networks
+        reseted_networks = []
+        for network in parents:
+            net = LinearQNetwork(self.nn_params, NEURAL_INPUT_SIZE, NEURAL_OUTPUT_SIZE)
+            net.network_params = network.network_params
+            reseted_networks.append(net)
+
+        parents = reseted_networks
+
+        # Randomly mutate some of the networks.
+        for parent in parents:
+            if self.mutation_chance > random.random():
+                parent = self.mutate(parent)
+
+        # Determine the number of freed spots for the next generation.
+        parents_length = len(parents)
+        desired_length = len(pop) - parents_length
+        children = []
+
+        # Fill missing spots with new children.
+        while len(children) < desired_length:
+            # Get random parents.
+            p1 = random.randint(0, parents_length - 1)
+            p2 = random.randint(0, parents_length - 1)
+
+            # Ensure they are not the same network.
+            if p1 != p2:
+                p1 = parents[p1]
+                p2 = parents[p2]
+
+                # Breed networks.
+                babies = self.breed(p1, p2)
+
+                # Add children one at a time.
+                for baby in babies:
+                    # Don't grow larger than the desired length.
+                    if len(children) < desired_length:
+                        children.append(baby)
+
+        parents.extend(children)
+        return parents

survival/ai/test.py

@@ -0,0 +1,93 @@
+import torch
+import pygad
+from pygad.torchga import torchga
+
+
+def fitness_func(solution, sol_idx):
+    global data_inputs, data_outputs, torch_ga, model, loss_function
+
+    model_weights_dict = torchga.model_weights_as_dict(model=model,
+                                                       weights_vector=solution)
+
+    # Use the current solution as the model parameters.
+    model.load_state_dict(model_weights_dict)
+
+    predictions = model(data_inputs)
+    abs_error = loss_function(predictions, data_outputs).detach().numpy() + 0.00000001
+
+    solution_fitness = 1.0 / abs_error
+
+    return solution_fitness
+
+def callback_generation(ga_instance):
+    print("Generation = {generation}".format(generation=ga_instance.generations_completed))
+    print("Fitness    = {fitness}".format(fitness=ga_instance.best_solution()[1]))
+
+# Create the PyTorch model.
+input_layer = torch.nn.Linear(3, 2)
+relu_layer = torch.nn.ReLU()
+output_layer = torch.nn.Linear(2, 1)
+
+model = torch.nn.Sequential(input_layer,
+                            relu_layer,
+                            output_layer)
+# print(model)
+
+# Create an instance of the pygad.torchga.TorchGA class to build the initial population.
+torch_ga = torchga.TorchGA(model=model,
+                           num_solutions=10)
+
+loss_function = torch.nn.L1Loss()
+
+# Data inputs
+data_inputs = torch.tensor([[0.02, 0.1, 0.15],
+                            [0.7, 0.6, 0.8],
+                            [1.5, 1.2, 1.7],
+                            [3.2, 2.9, 3.1]])
+
+# Data outputs
+data_outputs = torch.tensor([[0.1],
+                             [0.6],
+                             [1.3],
+                             [2.5]])
+
+# Prepare the PyGAD parameters. Check the documentation for more information: https://pygad.readthedocs.io/en/latest/README_pygad_ReadTheDocs.html#pygad-ga-class
+num_generations = 250 # Number of generations.
+num_parents_mating = 5 # Number of solutions to be selected as parents in the mating pool.
+initial_population = torch_ga.population_weights # Initial population of network weights
+parent_selection_type = "sss" # Type of parent selection.
+crossover_type = "single_point" # Type of the crossover operator.
+mutation_type = "random" # Type of the mutation operator.
+mutation_percent_genes = 10 # Percentage of genes to mutate. This parameter has no action if the parameter mutation_num_genes exists.
+keep_parents = -1 # Number of parents to keep in the next population. -1 means keep all parents and 0 means keep nothing.
+
+ga_instance = pygad.GA(num_generations=num_generations,
+                       num_parents_mating=num_parents_mating,
+                       initial_population=initial_population,
+                       fitness_func=fitness_func,
+                       parent_selection_type=parent_selection_type,
+                       crossover_type=crossover_type,
+                       mutation_type=mutation_type,
+                       mutation_percent_genes=mutation_percent_genes,
+                       keep_parents=keep_parents,
+                       on_generation=callback_generation)
+
+ga_instance.run()
+
+# After the generations complete, some plots are showed that summarize how the outputs/fitness values evolve over generations.
+ga_instance.plot_result(title="PyGAD & PyTorch - Iteration vs. Fitness", linewidth=4)
+
+# Returning the details of the best solution.
+solution, solution_fitness, solution_idx = ga_instance.best_solution()
+print("Fitness value of the best solution = {solution_fitness}".format(solution_fitness=solution_fitness))
+print("Index of the best solution : {solution_idx}".format(solution_idx=solution_idx))
+
+# Fetch the parameters of the best solution.
+best_solution_weights = torchga.model_weights_as_dict(model=model,
+                                                      weights_vector=solution)
+model.load_state_dict(best_solution_weights)
+predictions = model(data_inputs)
+print("Predictions : \n", predictions.detach().numpy())
+
+abs_error = loss_function(predictions, data_outputs)
+print("Absolute Error : ", abs_error.detach().numpy())

survival/components/consumption_component.py

@@ -0,0 +1,6 @@
+from survival.generators.resource_type import ResourceType
+
+
+class ConsumptionComponent:
+    def __init__(self):
+        self.status = {ResourceType.FOOD: 1, ResourceType.WOOD: 1, ResourceType.WATER: 1}

survival/components/direction_component.py

@@ -1,4 +1,4 @@
-from survival.enums import Direction
+from survival.game.enums import Direction
 
 
 class DirectionChangeComponent:

survival/components/inventory_component.py

@@ -1,5 +1,5 @@
 class InventoryComponent:
-    def __init__(self, maxitems=10):
+    def __init__(self, maxitems=100):
         self.maxitems = maxitems
         self.items = {}
 
@@ -17,5 +17,17 @@ class InventoryComponent:
             if self.items[item] < 0:
                 self.items[item] = 0
 
+    def count(self, item):
+        return self.items[item]
+
     def has_item(self, item):
         return item in self.items and self.items[item] != 0
+
+    def total_items_count(self):
+        total = 0
+        for item, value in self.items.items():
+            total += value
+        return total
+
+    def clear(self):
+        self.items = {}

survival/components/learning_component.py

@@ -0,0 +1,32 @@
+from survival.components.time_component import TimeComponent
+
+
+class LearningComponent:
+    def __init__(self):
+        self.made_step = False
+        self.old_state = None
+        self.action = None
+        self.resource = None
+
+        self.reward = 0
+        self.done = False
+        self.score = 0
+        self.record = 0
+
+    def load_step(self, old_state, action, resource):
+        self.old_state = old_state
+        self.action = action
+        if resource is None:
+            self.resource = None
+        else:
+            self.resource = resource
+        self.made_step = True
+
+    def reset(self):
+        self.made_step = False
+        self.old_state = None
+        self.action = None
+        self.resource = None
+
+        self.reward = 0
+        self.done = False

survival/components/on_collision_component.py

@@ -7,9 +7,9 @@ class OnCollisionComponent:
             callbacks = []
         self.callbacks = callbacks
 
-    def callAll(self):
+    def call_all(self):
         for func in self.callbacks:
             func()
 
-    def addCallback(self, fn, **kwargs):
+    def add_callback(self, fn, **kwargs):
         self.callbacks.append(partial(fn, **kwargs))

survival/components/position_component.py

@@ -1,4 +1,4 @@
-from survival.enums import Direction
+from survival.game.enums import Direction
 
 
 class PositionComponent:

survival/components/resource_component.py

@@ -1,3 +1,29 @@
+import random
+
+from survival.generators.resource_type import ResourceType
+
+
 class ResourceComponent:
     def __init__(self, resource_type):
         self.resource_type = resource_type
+        w, e, t = self.generate_attributes(resource_type)
+        self.weight = w
+        self.eatable = e
+        self.toughness = t
+
+    @staticmethod
+    def generate_attributes(resource_type):
+        if resource_type == ResourceType.WOOD:
+            weight = random.randint(10, 15)
+            eatable = False
+            toughness = random.randint(10, 15)
+        elif resource_type == ResourceType.WATER:
+            weight = random.randint(1, 2)
+            eatable = True
+            toughness = 0
+        else:
+            weight = random.randint(1, 7)
+            eatable = True
+            toughness = random.randint(2, 5)
+
+        return weight, eatable, toughness

survival/components/sprite_component.py

@@ -1,4 +1,4 @@
-from survival.image import Image
+from survival.game.image import Image
 
 
 class SpriteComponent:

survival/components/time_component.py

@@ -1,5 +1,5 @@
 class TimeComponent:
-    def __init__(self, minute, hour, day, timer):
+    def __init__(self, minute=0, hour=0, day=0, timer=0):
         self.minute = minute
         self.hour = hour
         self.day = day
@@ -16,5 +16,17 @@ class TimeComponent:
                 self.hour = temp2
             self.minute = temp
 
+    def total_minutes(self):
+        return self.minute + self.hour * 60 + self.day * 1440
+
     def __str__(self):
         return f'Day {self.day}, {self.hour}:{self.minute}'
+
+    def __eq__(self, other):
+        return self.total_minutes() == other.total_minutes()
+
+    def __gt__(self, other):
+        return self.total_minutes() > other.total_minutes()
+
+    def __lt__(self, other):
+        return self.total_minutes() < other.total_minutes()

survival/components/vision_component.py

@@ -0,0 +1,34 @@
+from pygame import Surface
+
+from survival.settings import AGENT_VISION_RANGE, SCREEN_WIDTH, SCREEN_HEIGHT
+
+
+class VisionComponent:
+    def __init__(self):
+        self.agent_vision = AGENT_VISION_RANGE * 32 * 2
+        self.width = SCREEN_WIDTH * 2
+        self.height = SCREEN_HEIGHT * 2
+        self.surface_l = Surface(((self.width - self.agent_vision) / 2, self.height))
+        self.surface_r = Surface(((self.width - self.agent_vision) / 2, self.height))
+        self.surface_t = Surface((self.agent_vision, (self.height - self.agent_vision) / 2))
+        self.surface_b = Surface((self.agent_vision, (self.height - self.agent_vision) / 2))
+        self.surface_l.fill((0, 0, 0))
+        self.surface_l.set_alpha(200)
+        self.surface_r.fill((0, 0, 0))
+        self.surface_r.set_alpha(200)
+        self.surface_t.fill((0, 0, 0))
+        self.surface_t.set_alpha(200)
+        self.surface_b.fill((0, 0, 0))
+        self.surface_b.set_alpha(200)
+        self.l_pos = (0, 0)
+        self.r_pos = (0, 0)
+        self.t_pos = (0, 0)
+        self.b_pos = (0, 0)
+
+    def update_positions(self, position: [int, int]):
+        new_position = (position[0] - self.width / 2 + 16, position[1] - self.height / 2 + 16)
+        self.l_pos = new_position
+        self.r_pos = (new_position[0] + (self.width + self.agent_vision) / 2, new_position[1])
+        self.t_pos = (new_position[0] + (self.width - self.agent_vision) / 2, new_position[1])
+        self.b_pos = (new_position[0] + (self.width - self.agent_vision) / 2,
+                            new_position[1] + (self.height + self.agent_vision) / 2)

survival/esper.py

@@ -28,6 +28,9 @@ class Processor:
     def process(self, *args, **kwargs):
         raise NotImplementedError
 
+    def reset(self, *args, **kwargs):
+        pass
+
 
 class World:
     """A World object keeps track of all Entities, Components, and Processors.
@@ -46,6 +49,14 @@ class World:
             self.process_times = {}
             self._process = self._timed_process
 
+    @property
+    def processors(self):
+        return self._processors
+
+    @property
+    def entities(self):
+        return self._entities
+
     def clear_cache(self) -> None:
         self.get_component.cache_clear()
         self.get_components.cache_clear()

survival/game/biomes/biome_data.py

@@ -1,4 +1,4 @@
-from survival.biomes.biome_preset import BiomePreset
+from survival.game.biomes.biome_preset import BiomePreset
 
 
 class BiomeData:

survival/game/biomes/biome_preset.py

@@ -1,7 +1,7 @@
 import random
 from typing import List
 
-from survival.tile import Tile
+from survival.game.tile import Tile
 
 
 class BiomePreset:

survival/game/camera.py

@@ -1,6 +1,6 @@
 from pygame.rect import Rect
 
-from survival import SCREEN_WIDTH, SCREEN_HEIGHT
+from survival.settings import SCREEN_WIDTH, SCREEN_HEIGHT
 
 
 class Camera:

survival/game/entity_layer.py

@@ -18,7 +18,7 @@ class EntityLayer:
     def remove_entity(self, pos):
         self.tiles[pos[1]][pos[0]] = None
 
-    def get_entity(self, pos) -> int:
+    def get_entity(self, pos):
         return self.tiles[pos[1]][pos[0]]
 
     def is_colliding(self, pos):

survival/game/game_map.py

@@ -0,0 +1,82 @@
+from survival.components.position_component import PositionComponent
+from survival.components.resource_component import ResourceComponent
+from survival.game.entity_layer import EntityLayer
+from survival.esper import World
+from survival.ai.graph_search import graph_search
+from survival.settings import AGENT_VISION_RANGE
+from survival.game.tile_layer import TileLayer
+
+
+class GameMap:
+    def __init__(self, width, height):
+        self.width = width
+        self.height = height
+        self.tile_layer = TileLayer(width, height)
+        self.entity_layer = EntityLayer(width, height)
+
+    def draw(self, camera):
+        visible_area = camera.get_visible_area()
+        self.tile_layer.draw(camera, visible_area)
+
+    def add_entity(self, entity, pos):
+        self.entity_layer.add_entity(entity, pos.grid_position)
+
+    def move_entity(self, from_pos, to_pos):
+        self.entity_layer.move_entity(from_pos, to_pos)
+
+    def remove_entity(self, pos):
+        self.entity_layer.remove_entity(pos)
+
+    def get_entity(self, pos) -> int:
+        if not self.in_bounds(pos):
+            return None
+        return self.entity_layer.get_entity(pos)
+
+    def is_colliding(self, pos):
+        return not self.in_bounds(pos) or self.entity_layer.is_colliding(pos)
+
+    def in_bounds(self, pos):
+        return 0 <= pos[0] < self.width and 0 <= pos[1] < self.height
+
+    def get_cost(self, pos):
+        return self.tile_layer.get_cost(pos)
+
+    def find_nearby_resources(self, world: World, player: int, position: PositionComponent, search_range: int = 5):
+        entity_position = position.grid_position
+
+        x_range = [entity_position[0] - search_range, entity_position[0] + search_range]
+        y_range = [entity_position[1] - search_range, entity_position[1] + search_range]
+
+        # Check if range is not out of map bounds
+        if x_range[0] < 0:
+            x_range[0] = 0
+        if x_range[1] >= self.width:
+            x_range[1] = self.width - 1
+        if y_range[0] < 0:
+            y_range[0] = 0
+        if y_range[1] >= self.height:
+            y_range[1] = self.height - 1
+
+        found_resources = []
+
+        for y in range(y_range[0], y_range[1]):
+            for x in range(x_range[0], x_range[1]):
+                ent = self.get_entity([x, y])
+                if ent == player:
+                    continue
+                if ent is not None and world.has_component(ent, ResourceComponent):
+                    res_position = world.component_for_entity(ent, PositionComponent).grid_position
+                    path, cost = graph_search(self, position, tuple(res_position), world)
+                    found_resources.append([ent, path, cost])
+
+        return found_resources
+
+    def find_nearest_resource(self, world: World, player: int, position: PositionComponent):
+        resources = self.find_nearby_resources(world, player, position, AGENT_VISION_RANGE)
+
+        nearest = None
+        for resource in resources:
+            if nearest is None or resource[2] < nearest[2]:
+                nearest = resource
+
+        return nearest

survival/game/image.py

@@ -4,8 +4,11 @@ import pygame
 
 
 class Image:
-    def __init__(self, filename, pos=(0, 0), scale=1):
-        self.texture = pygame.image.load(os.path.join('..', 'assets', filename)).convert_alpha()
+    def __init__(self, filename='', pos=(0, 0), scale=1, surface=None):
+        if surface is None:
+            self.texture = pygame.image.load(os.path.join('../', 'assets', filename)).convert_alpha()
+        else:
+            self.texture = surface
         self.image = self.texture
         self.origin = (0, 0)
         self.pos = pos

survival/game/tile_layer.py

@@ -1,6 +1,6 @@
 from survival.generators.tile_generator import TileGenerator
-from survival.image import Image
-from survival.tile import Tile
+from survival.game.image import Image
+from survival.game.tile import Tile
 
 
 class TileLayer:
@@ -8,7 +8,6 @@ class TileLayer:
         self.width = width
         self.height = height
         self.tiles: list[list[Tile]] = TileGenerator.generate_biome_tiles(width, height)
-        # self.tiles: list[list[Tile]] = TileGenerator.generate_random_tiles(width, height)
         self.image = Image('atlas.png')
 
     def draw(self, camera, visible_area):

survival/game/user_interface.py

@@ -1,19 +1,20 @@
 import pygame.font
 
-from survival import settings
+from survival.ai.genetic_algorithm import GeneticAlgorithm
+from survival.settings import MUTATE_NETWORKS, SCREEN_HEIGHT, SCREEN_WIDTH
 from survival.components.inventory_component import InventoryComponent
-from survival.generators.resource_generator import ResourceType
-from survival.image import Image
+from survival.generators.resource_type import ResourceType
+from survival.game.image import Image
 
 
 class UserInterface:
-    def __init__(self, window, inventory: InventoryComponent):
-        self.width = settings.SCREEN_WIDTH
-        self.height = settings.SCREEN_HEIGHT
+    def __init__(self, window):
+        self.width = SCREEN_WIDTH
+        self.height = SCREEN_HEIGHT
         self.window = window
         self.pos = (self.width - 240, 50)
         self.scale = 2
-        self.inventory = inventory
+        self.inventory: InventoryComponent = None
         self.images = {
             ResourceType.FOOD: Image('apple.png', self.pos, self.scale),
             ResourceType.WATER: Image('water.png', self.pos, self.scale),
@@ -25,6 +26,11 @@ class UserInterface:
             i += 1
         self.slot_image = Image('ui.png', self.pos, scale=2)
         self.font = pygame.font.SysFont('Comic Sans MS', 20)
+        self.initialized = False
+
+    def load_inventory(self, inventory: InventoryComponent):
+        self.inventory = inventory
+        self.initialized = True
 
     def update(self):
         pass
@@ -39,4 +45,3 @@ class UserInterface:
 
             textsurface = self.font.render(str(items_count), False, (255, 255, 255))
             self.window.blit(textsurface, (image.pos[0] + 48, image.pos[1] + 36))
-

survival/game_map.py

@@ -1,32 +0,0 @@
-from survival.entity_layer import EntityLayer
-from survival.tile_layer import TileLayer
-
-
-class GameMap:
-    def __init__(self, width, height):
-        self.width = width
-        self.height = height
-        self.tile_layer = TileLayer(width, height)
-        self.entity_layer = EntityLayer(width, height)
-
-    def draw(self, camera):
-        visible_area = camera.get_visible_area()
-        self.tile_layer.draw(camera, visible_area)
-
-    def add_entity(self, entity, pos):
-        self.entity_layer.add_entity(entity, pos.grid_position)
-
-    def move_entity(self, from_pos, to_pos):
-        self.entity_layer.move_entity(from_pos, to_pos)
-
-    def remove_entity(self, pos):
-        self.entity_layer.remove_entity(pos)
-
-    def get_entity(self, pos) -> int:
-        return self.entity_layer.get_entity(pos)
-
-    def is_colliding(self, pos):
-        return pos[0] < 0 or pos[0] >= self.width or pos[1] < 0 or pos[1] >= self.height or self.entity_layer.is_colliding(pos)
-
-    def get_cost(self, pos):
-        return self.tile_layer.get_cost(pos)

survival/generators/player_generator.py

@@ -1,29 +1,41 @@
-from survival.components.OnCollisionComponent import OnCollisionComponent
+from survival.components.on_collision_component import OnCollisionComponent
 from survival.components.camera_target_component import CameraTargetComponent
+from survival.components.consumption_component import ConsumptionComponent
 from survival.components.input_component import InputComponent
 from survival.components.inventory_component import InventoryComponent
+from survival.components.learning_component import LearningComponent
 from survival.components.movement_component import MovementComponent
 from survival.components.position_component import PositionComponent
 from survival.components.sprite_component import SpriteComponent
 from survival.components.time_component import TimeComponent
+from survival.components.vision_component import VisionComponent
+from survival.generators.resource_type import ResourceType
+from survival.settings import PLAYER_START_POSITION, STARTING_RESOURCES_AMOUNT
 
 
 class PlayerGenerator:
-
     def create_player(self, world, game_map):
         player = world.create_entity()
-        pos = PositionComponent([0, 0], [0, 0])
+        pos = PositionComponent([PLAYER_START_POSITION[0] * 32, PLAYER_START_POSITION[1] * 32],
+                                PLAYER_START_POSITION)
         world.add_component(player, pos)
         world.add_component(player, MovementComponent())
         world.add_component(player, InputComponent())
         world.add_component(player, OnCollisionComponent())
-        world.add_component(player, InventoryComponent())
+        inv = InventoryComponent()
+        for resource in ResourceType:
+            inv.add_item(resource, STARTING_RESOURCES_AMOUNT)
+        world.add_component(player, ConsumptionComponent())
+        world.add_component(player, inv)
         camera_target = CameraTargetComponent(pos)
         world.add_component(player, camera_target)
+        # world.add_component(player, AutomationComponent())
         game_map.add_entity(player, pos)
         sprite = SpriteComponent('stevenson.png')
         sprite.set_scale(1)
         world.add_component(player, sprite)
-        world.add_component(player, TimeComponent(0, 0, 0, 0))
+        world.add_component(player, TimeComponent())
+        world.add_component(player, VisionComponent())
+        world.add_component(player, LearningComponent())
 
         return player

survival/generators/resource_generator.py

@@ -1,28 +1,26 @@
 import random
 
-from enum import Enum
 from survival import GameMap
-from survival.components.OnCollisionComponent import OnCollisionComponent
+from survival.components.on_collision_component import OnCollisionComponent
 from survival.components.inventory_component import InventoryComponent
+from survival.components.learning_component import LearningComponent
 from survival.components.position_component import PositionComponent
 from survival.components.resource_component import ResourceComponent
 from survival.components.sprite_component import SpriteComponent
 from survival.esper import World
-from survival.settings import RESOURCES_AMOUNT
-
-
-class ResourceType(Enum):
-    FOOD = 1
-    WATER = 2
-    WOOD = 3
+from survival.generators.resource_type import ResourceType
+from survival.settings import RESOURCES_AMOUNT, PLAYER_START_POSITION
 
 
 class ResourceGenerator:
+    resources_amount = 0
+
     def __init__(self, world, game_map):
         self.world = world
         self.map = game_map
 
     def generate_resources(self, player: int):
+        ResourceGenerator.resources_amount = RESOURCES_AMOUNT
         for x in range(RESOURCES_AMOUNT):
             obj = self.world.create_entity()
             sprites = {
@@ -37,7 +35,7 @@ class ResourceGenerator:
             resource_type = random.choice(list(ResourceType))
             sprite = SpriteComponent(sprites[resource_type])
             col = OnCollisionComponent()
-            col.addCallback(self.remove_resource, world=self.world, game_map=self.map, entity=obj, player=player)
+            col.add_callback(self.remove_resource, world=self.world, game_map=self.map, resource_ent=obj, player=player)
             self.world.add_component(obj, pos)
             self.world.add_component(obj, sprite)
             self.world.add_component(obj, col)
@@ -46,15 +44,25 @@ class ResourceGenerator:
 
     def get_empty_grid_position(self):
         free_pos = [random.randrange(self.map.width), random.randrange(self.map.height)]
-        while self.map.is_colliding(free_pos):
+        while self.map.is_colliding(free_pos) or (
+                free_pos[0] == PLAYER_START_POSITION[0] and free_pos[1] == PLAYER_START_POSITION[1]):
             free_pos = [random.randrange(self.map.width), random.randrange(self.map.height)]
         return free_pos
 
     @staticmethod
-    def remove_resource(world: World, game_map: GameMap, entity: int, player: int):
-        pos = world.component_for_entity(entity, PositionComponent)
-        resource = world.component_for_entity(entity, ResourceComponent)
+    def remove_resource(world: World, game_map: GameMap, resource_ent: int, player: int):
+        pos = world.component_for_entity(resource_ent, PositionComponent)
+        resource = world.component_for_entity(resource_ent, ResourceComponent)
         inventory = world.component_for_entity(player, InventoryComponent)
         inventory.add_item(resource.resource_type, 1)
         game_map.remove_entity(pos.grid_position)
-        world.delete_entity(entity, immediate=True)
+        world.delete_entity(resource_ent, immediate=True)
+        if world.has_component(player, LearningComponent):
+            learning = world.component_for_entity(player, LearningComponent)
+            learning.reward += 10
+            learning.score += 1
+            ResourceGenerator.resources_amount -= 1
+            if ResourceGenerator.resources_amount == 0:
+                learning.reward += 50
+                learning.done = True
+

survival/generators/resource_type.py

@@ -0,0 +1,18 @@
+from enum import Enum
+
+
+class ResourceType(Enum):
+    FOOD = 1
+    WATER = 2
+    WOOD = 3
+
+    @staticmethod
+    def get_from_string(string):
+        if string == 'food':
+            return ResourceType.FOOD
+        elif string == 'water':
+            return ResourceType.WATER
+        elif string == 'wood':
+            return ResourceType.WOOD
+        else:
+            raise Exception("Unknown resource type")

survival/generators/tile_generator.py

@@ -1,10 +1,12 @@
+import json
 import random
+from pathlib import Path
 from typing import List
 
-from survival.biomes.biome_data import BiomeData
-from survival.biomes.biome_preset import BiomePreset
-from survival.biomes.noise import generate_noise
-from survival.tile import Tile
+from survival.game.biomes.biome_data import BiomeData
+from survival.game.biomes.biome_preset import BiomePreset
+from survival.game.biomes.noise import generate_noise
+from survival.game.tile import Tile
 
 
 class TileGenerator:
@@ -44,11 +46,25 @@ class TileGenerator:
 
     @staticmethod
     def generate_biome_tiles(width: int, height: int):
-        seed = random.randint(0, 9999999)
+        # Use static seed to allow smooth learning of genetic algorithm
+        seed = 1
         octaves = 10
-        height_map = generate_noise(width, height, octaves, seed)
-        moisture_map = generate_noise(width, height, octaves, seed)
-        heat_map = generate_noise(width, height, octaves, seed)
+        file_name = f'seeds/{seed}.bin'
+        biomes_file = Path(file_name)
+        if biomes_file.is_file():
+            with open(file_name, 'r') as f:
+                data = json.load(f)
+                height_map = data[0]
+                moisture_map = data[1]
+                heat_map = data[2]
+        else:
+            height_map = generate_noise(width, height, octaves, seed)
+            moisture_map = generate_noise(width, height, octaves, seed)
+            heat_map = generate_noise(width, height, octaves, seed)
+            data = [height_map, moisture_map, heat_map]
+            Path('seeds').mkdir(exist_ok=True)
+            with open(file_name, 'w') as f:
+                json.dump(data, f)
 
         return [[TileGenerator.get_biome(height_map[y][x], moisture_map[y][x], heat_map[y][x]).get_new_tile() for x in
                  range(width)] for y in range(height)]

survival/generators/world_generator.py

@@ -1,25 +1,119 @@
-from survival import esper
+from pathlib import Path
+
+from survival import esper, ResourceGenerator, PlayerGenerator
+from survival.ai.model import LinearQNetwork
+from survival.components.consumption_component import ConsumptionComponent
+from survival.components.direction_component import DirectionChangeComponent
+from survival.components.inventory_component import InventoryComponent
+from survival.components.learning_component import LearningComponent
+from survival.components.moving_component import MovingComponent
+from survival.components.pathfinding_component import PathfindingComponent
+from survival.components.position_component import PositionComponent
+from survival.components.resource_component import ResourceComponent
+from survival.components.time_component import TimeComponent
+from survival.esper import World
+from survival.game.camera import Camera
+from survival.game.game_map import GameMap
+from survival.generators.resource_type import ResourceType
+from survival.settings import PLAYER_START_POSITION, STARTING_RESOURCES_AMOUNT, SCREEN_WIDTH, SCREEN_HEIGHT, \
+    MUTATE_NETWORKS, NETWORK_PARAMS, NEURAL_OUTPUT_SIZE, NEURAL_INPUT_SIZE
+from survival.systems.automation_system import AutomationSystem
 from survival.systems.camera_system import CameraSystem
 from survival.systems.collision_system import CollisionSystem
+from survival.systems.consumption_system import ConsumptionSystem
 from survival.systems.direction_system import DirectionSystem
 from survival.systems.draw_system import DrawSystem
 from survival.systems.input_system import InputSystem
 from survival.systems.movement_system import MovementSystem
-from survival.systems.pathfinding_movement_system import PathfindingMovementSystem
+from survival.systems.neural_system import NeuralSystem
 from survival.systems.time_system import TimeSystem
+from survival.systems.vision_system import VisionSystem
 
 
 class WorldGenerator:
+    def __init__(self, win, callback):
+        self.win = win
+        self.callback = callback
+        self.world: World = esper.World(timed=True)
+        self.game_map: GameMap = GameMap(int(SCREEN_WIDTH / 32) * 2, 2 * int(SCREEN_HEIGHT / 32) + 1)
+        self.camera = Camera(self.game_map.width * 32, self.game_map.height * 32, self.win)
+        self.resource_generator: ResourceGenerator = ResourceGenerator(self.world, self.game_map)
+        self.player: int = -1
 
-    def create_world(self, camera, game_map):
-        world = esper.World()
-        world.add_processor(InputSystem(camera))
-        world.add_processor(CameraSystem(camera))
-        world.add_processor(MovementSystem(game_map), priority=1)
-        world.add_processor(CollisionSystem(game_map), priority=2)
-        world.add_processor(DrawSystem(camera))
-        world.add_processor(TimeSystem())
-        world.add_processor(PathfindingMovementSystem(game_map), priority=3)
-        world.add_processor(DirectionSystem())
+    def create_world(self):
+        self.world.add_processor(InputSystem(self.camera, self.game_map))
+        self.world.add_processor(CameraSystem(self.camera))
+        self.world.add_processor(MovementSystem(self.game_map), priority=20)
+        self.world.add_processor(CollisionSystem(self.game_map), priority=30)
+        self.world.add_processor(NeuralSystem(self.game_map, self.callback), priority=50)
+        if not MUTATE_NETWORKS:
+            model_path = Path("/model/model.pth")
+            if model_path.is_file():
+                self.world.get_processor(NeuralSystem).load_model(
+                    LinearQNetwork.load(NETWORK_PARAMS, NEURAL_INPUT_SIZE, NEURAL_OUTPUT_SIZE))
+            else:
+                self.world.get_processor(NeuralSystem).load_model(
+                    LinearQNetwork(NETWORK_PARAMS, NEURAL_INPUT_SIZE, NEURAL_OUTPUT_SIZE, False, NETWORK_PARAMS))
+        self.world.add_processor(DrawSystem(self.camera))
+        self.world.add_processor(TimeSystem())
+        self.world.add_processor(AutomationSystem(self.game_map))
+        # self.world.add_processor(PathfindingMovementSystem(self.game_map), priority=40)
+        self.world.add_processor(DirectionSystem())
+        self.world.add_processor(ConsumptionSystem(self.callback))
+        self.world.add_processor(VisionSystem(self.camera))
 
-        return world
+        self.player = PlayerGenerator().create_player(self.world, self.game_map)
+        self.world.get_processor(DrawSystem).initialize_interface(
+            self.world.component_for_entity(self.player, InventoryComponent))
+
+        # BuildingGenerator().create_home(self.world, self.game_map)
+        self.resource_generator.generate_resources(self.player)
+        return self.game_map, self.world, self.camera
+
+    def reset_world(self):
+        for processor in self.world.processors:
+            processor.reset()
+
+        self.reset_player()
+        self.reset_resources()
+
+    def reset_resources(self):
+        for entity in self.world.entities:
+            if self.world.has_component(entity, ResourceComponent):
+                self.game_map.remove_entity(self.world.component_for_entity(entity, PositionComponent).grid_position)
+                self.world.delete_entity(entity)
+                continue
+        self.resource_generator.generate_resources(self.player)
+
+    def reset_player(self):
+        self.world.remove_component(self.player, TimeComponent)
+        self.world.add_component(self.player, TimeComponent())
+
+        inv = self.world.component_for_entity(self.player, InventoryComponent)
+        inv.clear()
+        for resource in ResourceType:
+            inv.add_item(resource, STARTING_RESOURCES_AMOUNT)
+
+        if self.world.has_component(self.player, ConsumptionComponent):
+            self.world.remove_component(self.player, ConsumptionComponent)
+            self.world.add_component(self.player, ConsumptionComponent())
+
+        pos = self.world.component_for_entity(self.player, PositionComponent)
+        old_pos = pos.grid_position
+
+        self.world.remove_component(self.player, PositionComponent)
+        self.world.add_component(self.player,
+                                 PositionComponent([PLAYER_START_POSITION[0] * 32, PLAYER_START_POSITION[1] * 32],
+                                                   PLAYER_START_POSITION))
+
+        self.game_map.move_entity(old_pos, pos.grid_position)
+
+        if self.world.has_component(self.player, MovingComponent):
+            self.world.remove_component(self.player, MovingComponent)
+        if self.world.has_component(self.player, DirectionChangeComponent):
+            self.world.remove_component(self.player, DirectionChangeComponent)
+        if self.world.has_component(self.player, PathfindingComponent):
+            self.world.remove_component(self.player, PathfindingComponent)
+        if self.world.has_component(self.player, LearningComponent):
+            learning = self.world.component_for_entity(self.player, LearningComponent)
+            learning.reset()

survival/player.py

@@ -1,78 +0,0 @@
-from random import randint
-
-import pygame
-
-
-class Player:
-    def __init__(self):
-        # self.pos = [1024, 512]
-        # self.velocity = [0, 0]
-        # self.image = Image('stevenson.png')
-        # self.image.set_scale(2)
-        # self.speed = 30
-        # self.movement_target = [self.pos[0], self.pos[1]]
-        # self.timer = 0
-        pass
-
-    def draw(self, camera):
-
-        self.image.pos = self.pos
-        camera.draw(self.image)
-
-    def is_moving(self):
-        return self.pos != self.movement_target
-
-    def move_in_random_direction(self):
-        value = randint(0, 3)
-        random_movement = {
-            0: self.move_up,
-            1: self.move_down,
-            2: self.move_left,
-            3: self.move_right
-        }
-        random_movement[value]()
-
-    def update(self, delta, pressed_keys):
-        if self.is_moving():
-            if self.velocity[0] != 0:
-                self.pos[0] += self.velocity[0] * self.speed * delta / 100
-                if abs(self.movement_target[0] - self.pos[0]) < 0.1 * self.speed:
-                    self.velocity = [0, 0]
-                    self.pos = self.movement_target
-            else:
-                self.pos[1] += self.velocity[1] * self.speed * delta / 100
-                if abs(self.pos[1] - self.movement_target[1]) < 0.1 * self.speed:
-                    self.velocity = [0, 0]
-                    self.pos = self.movement_target
-            return
-
-        self.timer += delta
-
-        if self.timer > 1000:
-            self.move_in_random_direction()
-            self.timer = 0
-
-        if pressed_keys[pygame.K_LEFT]:
-            self.move_left()
-        elif pressed_keys[pygame.K_RIGHT]:
-            self.move_right()
-        elif pressed_keys[pygame.K_DOWN]:
-            self.move_down()
-        elif pressed_keys[pygame.K_UP]:
-            self.move_up()
-
-    def move_left(self):
-        self.velocity = [-1, 0]
-        self.movement_target = [self.pos[0] - 32, self.pos[1]]
-
-    def move_right(self):
-        self.velocity = [1, 0]
-        self.movement_target = [self.pos[0] + 32, self.pos[1]]
-
-    def move_up(self):
-        self.velocity = [0, -1]
-        self.movement_target = [self.pos[0], self.pos[1] - 32]
-
-    def move_down(self):
-        self.velocity = [0, 1]
-        self.movement_target = [self.pos[0], self.pos[1] + 32]

survival/settings.py

@@ -1,4 +1,18 @@
 SCREEN_WIDTH = 1000
 SCREEN_HEIGHT = 600
-RESOURCES_AMOUNT = 300
-DIRECTION_CHANGE_DELAY = 200
+RESOURCES_AMOUNT = 175
+DIRECTION_CHANGE_DELAY = 5
+PLAYER_START_POSITION = [20, 10]
+STARTING_RESOURCES_AMOUNT = 5
+AGENT_VISION_RANGE = 5
+NEURAL_INPUT_SIZE = 11
+NEURAL_OUTPUT_SIZE = 3
+LEARN = True
+MUTATE_NETWORKS = True
+NETWORK_PARAMS = {
+    "neurons": 256,
+    "layers": 1,
+    "activation": 'relu',
+    "ratio": 0.001,
+    "optimizer": 'Adam'
+}

survival/systems/automation_system.py

@@ -0,0 +1,62 @@
+from survival import esper, GameMap
+from survival.components.moving_component import MovingComponent
+from survival.components.pathfinding_component import PathfindingComponent
+from survival.components.position_component import PositionComponent
+from survival.components.resource_component import ResourceComponent
+
+
+class AutomationComponent:
+    pass
+
+
+class AutomationSystem(esper.Processor):
+    def __init__(self, game_map: GameMap):
+        self.game_map = game_map
+
+    def process(self, dt):
+        for ent, (automation, pos) in self.world.get_components(AutomationComponent, PositionComponent):
+            if self.world.has_component(ent, PathfindingComponent):
+                continue
+
+            resource = self.detect_closest_resource(pos, ent)
+            if resource is None:
+                # TODO: Check if target position is not out of map bounds
+                self.world.add_component(ent, PathfindingComponent((pos.grid_position[0] * 32 + 64, pos.grid_position[1] * 32 + 64)))
+                # Move somewhere else
+            else:
+                target = self.world.component_for_entity(resource, PositionComponent).grid_position
+                self.world.add_component(ent, PathfindingComponent((target[0] * 32, target[1] * 32), True))
+                # Go collect target resource
+
+    def detect_closest_resource(self, position: PositionComponent, target_entity: int):
+        entity_position = position.grid_position
+        x_range = [entity_position[0] - 5, entity_position[0] + 5]
+        y_range = [entity_position[1] - 5, entity_position[1] + 5]
+
+        # Check if range is not out of map bounds
+        if x_range[0] < 0:
+            x_range[0] = 0
+        if x_range[1] >= self.game_map.width:
+            x_range[1] = self.game_map.width - 1
+        if y_range[0] < 0:
+            y_range[0] = 0
+        if y_range[1] >= self.game_map.height:
+            y_range[1] = self.game_map.height - 1
+
+        found_resource = [-1, 200000]
+
+        for y in range(y_range[0], y_range[1]):
+            for x in range(x_range[0], x_range[1]):
+                ent = self.game_map.get_entity([x, y])
+                if ent == target_entity:
+                    continue
+                if ent is not None and self.world.has_component(ent, ResourceComponent):
+                    res_position = self.world.component_for_entity(ent, PositionComponent).grid_position
+                    distance = abs(entity_position[0] - res_position[0]) + abs(entity_position[1] - res_position[1])
+                    if found_resource[1] > distance:
+                        found_resource = [ent, distance]
+
+        if found_resource[0] == -1:
+            return None
+        else:
+            return found_resource[0]

survival/systems/collision_system.py

@@ -1,10 +1,11 @@
 import operator
 
 from survival import esper
-from survival.components.OnCollisionComponent import OnCollisionComponent
+from survival.components.on_collision_component import OnCollisionComponent
 from survival.components.moving_component import MovingComponent
 from survival.components.position_component import PositionComponent
-from survival.enums import Direction
+from survival.game.enums import Direction
+from survival.systems.consumption_system import ConsumeComponent
 
 
 class CollisionSystem(esper.Processor):
@@ -18,23 +19,24 @@ class CollisionSystem(esper.Processor):
                 continue
 
             moving.checked_collision = True
-
             vector = Direction.get_vector(pos.direction)
             moving.target = tuple(map(operator.add, vector, pos.grid_position))
             moving.direction_vector = vector
             if self.check_collision(moving.target):
+                self.world.add_component(ent, ConsumeComponent(0.05))
                 self.world.remove_component(ent, MovingComponent)
-                onCol.callAll()
+                onCol.call_all()
                 colliding_object: int = self.map.get_entity(moving.target)
 
                 if colliding_object is None or not self.world.entity_exists(colliding_object):
                     continue
 
                 if self.world.has_component(colliding_object, OnCollisionComponent):
-                    self.world.component_for_entity(colliding_object, OnCollisionComponent).callAll()
+                    self.world.component_for_entity(colliding_object, OnCollisionComponent).call_all()
 
             else:
                 self.map.move_entity(pos.grid_position, moving.target)
+                self.world.add_component(ent, ConsumeComponent(self.map.get_cost(moving.target)))
                 pos.grid_position = moving.target
 
     def check_collision(self, pos):

survival/systems/consumption_system.py

@@ -0,0 +1,64 @@
+import random
+
+from survival import esper
+from survival.components.consumption_component import ConsumptionComponent
+from survival.components.inventory_component import InventoryComponent
+from survival.components.learning_component import LearningComponent
+from survival.components.moving_component import MovingComponent
+from survival.generators.resource_type import ResourceType
+
+
+class ConsumeComponent:
+    def __init__(self, cost):
+        self.cost = cost
+
+
+class ConsumptionSystem(esper.Processor):
+    CONSUMPTION_FACTOR = 0.05
+    CONSUMPTION_RANGE = 0.07
+
+    def __init__(self, callback):
+        self.callback = callback
+
+    def process(self, dt):
+        cons: ConsumptionComponent
+        inventory: InventoryComponent
+        c: ConsumeComponent
+        for ent, (cons, inventory, c) in self.world.get_components(ConsumptionComponent, InventoryComponent,
+                                                                   ConsumeComponent):
+            for resource in cons.status.keys():
+                cons.status[resource] -= c.cost * self.CONSUMPTION_FACTOR + random.uniform(-self.CONSUMPTION_RANGE,
+                                                                                           self.CONSUMPTION_RANGE)
+                if cons.status[resource] < 0:
+                    inventory.items[resource] -= 1
+                    cons.status[resource] = 1
+
+            if self.world.has_component(ent, LearningComponent):
+                for resource in cons.status.keys():
+                    if inventory.items[resource] <= 0 and self.world.has_component(ent, LearningComponent):
+                        # If entity has run out of items
+                        learning: LearningComponent = self.world.component_for_entity(ent, LearningComponent)
+                        learning.reward -= 1
+                        learning.done = True
+                        break
+            else:
+                self.callback(ent)
+
+            self.world.remove_component(ent, ConsumeComponent)
+            # cons.timer -= dt
+            # if cons.timer > 0:
+            #     continue
+            # cons.timer = cons.timer_value
+            #
+            # if self.world.has_component(ent, LearningComponent):
+            #     # If no item was picked up
+            #     if cons.last_inventory_state == inventory.total_items_count():
+            #         learning: LearningComponent = self.world.component_for_entity(ent, LearningComponent)
+            #         learning.reward += -10
+            #         learning.done = True
+            #     cons.last_inventory_state = inventory.total_items_count()
+            # else:
+            #     if inventory.has_item(ResourceType.FOOD):
+            #         inventory.remove_item(ResourceType.FOOD, 1)
+            #     else:
+            #         self.callback()

survival/systems/draw_system.py

@@ -1,21 +1,22 @@
 from survival import esper
 from survival.components.position_component import PositionComponent
 from survival.components.sprite_component import SpriteComponent
-from survival.user_interface import UserInterface
+from survival.game.user_interface import UserInterface
 
 
 class DrawSystem(esper.Processor):
     def __init__(self, camera):
         self.camera = camera
-        self.ui = None
+        self.ui = UserInterface(self.camera.window)
 
     def initialize_interface(self, inventory):
-        self.ui = UserInterface(self.camera.window, inventory)
+        self.ui.load_inventory(inventory)
 
     def process(self, dt):
         for ent, (sprite, pos) in self.world.get_components(SpriteComponent, PositionComponent):
             sprite.image.pos = pos.position
             sprite.image.origin = (32 * pos.direction.value, 0)
             self.camera.draw(sprite.image)
-            self.ui.update()
-            self.ui.draw()
+            if self.ui.initialized:
+                self.ui.update()
+                self.ui.draw()

survival/systems/input_system.py

@@ -1,16 +1,18 @@
 import pygame
 
-from survival import esper
+from survival import esper, GameMap
 from survival.components.direction_component import DirectionChangeComponent
 from survival.components.input_component import InputComponent
 from survival.components.moving_component import MovingComponent
 from survival.components.pathfinding_component import PathfindingComponent
 from survival.components.position_component import PositionComponent
+from survival.components.resource_component import ResourceComponent
 
 
 class InputSystem(esper.Processor):
-    def __init__(self, camera):
+    def __init__(self, camera, game_map: GameMap):
         self.camera = camera
+        self.game_map = game_map
 
     def process(self, dt):
         for ent, (inp, pos) in self.world.get_components(InputComponent, PositionComponent):
@@ -20,7 +22,11 @@ class InputSystem(esper.Processor):
                 pos = pygame.mouse.get_pos()
                 pos = (pos[0] - self.camera.camera.left, pos[1] - self.camera.camera.top)
                 if not self.world.has_component(ent, PathfindingComponent):
-                    self.world.add_component(ent, PathfindingComponent(pos))
+                    target_ent = self.game_map.get_entity([int(pos[0] / 32), int(pos[1]/ 32)])
+                    if target_ent is not None and self.world.has_component(target_ent, ResourceComponent):
+                        self.world.add_component(ent, PathfindingComponent(pos))
+                    else:
+                        self.world.add_component(ent, PathfindingComponent(pos))
 
             if self.world.has_component(ent, MovingComponent):
                 continue

survival/systems/movement_system.py

@@ -13,11 +13,13 @@ class MovementSystem(esper.Processor):
         for ent, (mov, pos, moving, sprite) in self.world.get_components(MovementComponent, PositionComponent,
                                                                          MovingComponent,
                                                                          SpriteComponent):
-            cost = self.map.get_cost(moving.target)
-            pos.position[0] += moving.direction_vector[0] * mov.speed * dt    / 100 / cost
-            pos.position[1] += moving.direction_vector[1] * mov.speed * dt   / 100 / cost
-
-            if abs(moving.target[0] * 32 - pos.position[0]) < 0.1 * mov.speed and abs(
-                    pos.position[1] - moving.target[1] * 32) < 0.1 * mov.speed:
-                pos.position = [moving.target[0] * 32, moving.target[1] * 32]
-                self.world.remove_component(ent, MovingComponent)
+            # cost = self.map.get_cost(moving.target)
+            # pos.position[0] += moving.direction_vector[0] * mov.speed * dt / 100 / cost
+            # pos.position[1] += moving.direction_vector[1] * mov.speed * dt / 100 / cost
+            #
+            # if abs(moving.target[0] * 32 - pos.position[0]) < 1 * mov.speed and abs(
+            #         pos.position[1] - moving.target[1] * 32) < 1 * mov.speed:
+            #     pos.position = [moving.target[0] * 32, moving.target[1] * 32]
+            #     self.world.remove_component(ent, MovingComponent)
+            pos.position = [moving.target[0] * 32, moving.target[1] * 32]
+            self.world.remove_component(ent, MovingComponent)

survival/systems/neural_system.py

@@ -0,0 +1,142 @@
+import random
+from collections import deque
+
+import torch
+
+from survival import esper, GameMap
+from survival.ai.genetic_algorithm import GeneticAlgorithm
+from survival.components.direction_component import DirectionChangeComponent
+from survival.components.inventory_component import InventoryComponent
+from survival.components.moving_component import MovingComponent
+from survival.components.position_component import PositionComponent
+from survival.components.learning_component import LearningComponent
+from survival.components.time_component import TimeComponent
+from survival.ai.graph_search import Action
+from survival.ai.learning_utils import get_state, LearningUtils
+from survival.ai.model import LinearQNetwork, QTrainer
+from survival.settings import LEARN, MUTATE_NETWORKS
+
+MAX_MEMORY = 100_000
+BATCH_SIZE = 1000
+
+
+class NeuralSystem(esper.Processor):
+    def __init__(self, game_map: GameMap, callback):
+        self.game_map = game_map
+        self.reset_game = callback
+        self.n_games = 0  # number of games played
+        if MUTATE_NETWORKS:
+            self.starting_epsilon = GeneticAlgorithm.GAMES_PER_NETWORK / 2
+        else:
+            self.starting_epsilon = 100
+        self.epsilon = 0  # controlls the randomness
+        self.gamma = 0.9  # discount rate
+        self.memory = deque(maxlen=MAX_MEMORY)  # exceeding memory removes the left elements to make more space
+        self.model = None  # self.model = LinearQNetwork.load(11, 256, 3)
+        self.trainer = None  # QTrainer(self.model, lr=LR, gamma=self.gamma)
+        self.utils = LearningUtils()
+        self.best_action = None
+
+    def load_model(self, model: LinearQNetwork):
+        self.model = model
+        self.trainer = QTrainer(self.model, self.model.network_params['ratio'], self.gamma,
+                                self.model.network_params['optimizer'])
+        self.utils = LearningUtils()
+        self.memory = deque(maxlen=MAX_MEMORY)
+        self.starting_epsilon = GeneticAlgorithm.GAMES_PER_NETWORK / 2
+        self.n_games = 0
+
+    def remember(self, state, action, reward, next_state, done):
+        self.memory.append((state, action, reward, next_state, done))
+
+    def train_short_memory(self, state, action, reward, next_state, done):
+        self.trainer.train_step(state, action, reward, next_state, done)
+
+    def train_long_memory(self):
+        if len(self.memory) > BATCH_SIZE:
+            mini_sample = random.sample(self.memory, BATCH_SIZE)
+        else:
+            mini_sample = self.memory
+        states, actions, rewards, next_states, dones = zip(*mini_sample)
+        self.trainer.train_step(states, actions, rewards, next_states, dones)
+
+    def get_action(self, state):
+        self.epsilon = self.starting_epsilon - self.n_games
+        final_move = [0, 0, 0]
+        if random.randint(0, 200) < self.epsilon:
+            move = random.randint(0, 2)
+            final_move[move] = 1
+        else:
+            state_zero = torch.tensor(state, dtype=torch.float)
+            prediction = self.model(state_zero)
+            move = torch.argmax(prediction).item()
+            final_move[move] = 1
+
+        return final_move
+
+    def process(self, dt):
+        for ent, (pos, inventory, time, learning) in self.world.get_components(PositionComponent, InventoryComponent,
+                                                                               TimeComponent, LearningComponent):
+            if not learning.made_step:
+                learning.reset()
+                self.best_action = None
+
+                # Get the closest resource | [entity, path, cost]
+                resource: [int, list, int] = self.game_map.find_nearest_resource(self.world, ent, pos)
+
+                if resource is not None:
+                    # If resource was found get the best move chosen by A*
+                    self.best_action = resource[1][0]
+
+                # Get current entity state
+                old_state = get_state(self, ent, resource)
+                # Predict the action
+                action = self.get_action(old_state)
+                # Save the action
+                learning.load_step(old_state, action, resource)
+                # Perform the action
+                act = Action.perform(self.world, ent, Action.from_array(action))
+                self.utils.append_action(act, pos)
+
+                # Add reward if chosen action was the best action
+                if act == self.best_action:
+                    learning.reward += 1
+                continue
+
+            # Wait for the action to complete
+            if self.world.has_component(ent, DirectionChangeComponent) or self.world.has_component(ent,
+                                                                                                   MovingComponent):
+                continue
+
+            self.utils.check_last_actions(learning)
+
+            resource = learning.resource
+            if resource is None or not self.world.entity_exists(resource[0]):
+                # Find a new resource if no resource was found or the last one was consumed
+                resource = self.game_map.find_nearest_resource(self.world, ent, pos)
+
+            # Get new state
+            new_state = get_state(self, ent, resource)
+            # Train agent's memory
+            self.train_short_memory(learning.old_state, learning.action, learning.reward, new_state, learning.done)
+            self.remember(learning.old_state, learning.action, learning.reward, new_state, learning.done)
+
+            learning.made_step = False
+
+            if learning.done:
+                self.n_games += 1
+                if LEARN:
+                    self.train_long_memory()
+                if learning.score > learning.record:
+                    learning.record = learning.score
+                    if LEARN and not MUTATE_NETWORKS:
+                        self.model.save()
+
+                # print('Game', self.n_games, 'Score', learning.score, 'Record', learning.record)
+                self.utils.add_scores(learning, self.n_games)
+
+                self.model.scores.append(learning.score)
+                learning.score = 0
+                self.utils.plot()
+
+                self.reset_game()

survival/systems/pathfinding_movement_system.py

@@ -3,21 +3,20 @@ from survival.components.direction_component import DirectionChangeComponent
 from survival.components.movement_component import MovementComponent
 from survival.components.moving_component import MovingComponent
 from survival.components.position_component import PositionComponent
-from survival.enums import Direction
-from survival.graph_search import graph_search, Action
+from survival.game.enums import Direction
+from survival.ai.graph_search import graph_search, Action
 from survival.systems.input_system import PathfindingComponent
 
 
 class PathfindingMovementSystem(esper.Processor):
     def __init__(self, game_map):
         self.game_map = game_map
-        pass
 
     def process(self, dt):
         for ent, (pos, pathfinding, movement) in self.world.get_components(PositionComponent, PathfindingComponent,
                                                                            MovementComponent):
             if pathfinding.path is None:
-                pathfinding.path = graph_search(self.game_map, pos, pathfinding.target_grid_pos)
+                pathfinding.path, cost = graph_search(self.game_map, pos, pathfinding.target_grid_pos, self.world)
 
             if len(pathfinding.path) < 1:
                 self.world.remove_component(ent, PathfindingComponent)

survival/systems/time_system.py

@@ -9,4 +9,3 @@ class TimeSystem(esper.Processor):
             if time.timer > 1000:
                 time.add_time(1)
                 time.timer = 0
-                print(time)

survival/systems/vision_system.py

@@ -0,0 +1,18 @@
+from survival import esper
+from survival.components.position_component import PositionComponent
+from survival.components.vision_component import VisionComponent
+
+
+class VisionSystem(esper.Processor):
+    def __init__(self, camera):
+        self.camera = camera
+
+    def process(self, dt):
+        pos: PositionComponent
+        vision: VisionComponent
+        for ent, (pos, vision) in self.world.get_components(PositionComponent, VisionComponent):
+            vision.update_positions(pos.position)
+            self.camera.window.blit(vision.surface_l, self.camera.apply(vision.l_pos))
+            self.camera.window.blit(vision.surface_r, self.camera.apply(vision.r_pos))
+            self.camera.window.blit(vision.surface_t, self.camera.apply(vision.t_pos))
+            self.camera.window.blit(vision.surface_b, self.camera.apply(vision.b_pos))