refactor: remove unused import of time module in main.py

app/.dockerignore

@@ -17,7 +17,7 @@ logs/
 
 # Ignore Docker files
 Dockerfile
-docker-compose.yml
+docker-compose.yaml
 
 # Ignore build directories
 dist/

app/docker-compose.yaml

@@ -1,18 +1,15 @@
 services:
   webscraper:
     build:
-      context: ./app
+      context: .
+      dockerfile: ./docker/scripts/Dockerfile
     container_name: webscraper
     volumes:
-      - ./app:/usr/src/app
-    develop:
-      watch:
-        - path: ./app/requirements.txt
-          action: rebuild
-        - path: ./app
-          target: /usr/src/app
-          action: sync+restart
-    command: tail -f /dev/null
+      - .:/usr/src/app
+    command:
+      - tail
+      - -f
+      - /dev/null
   selenium-hub:
     image: "selenium/hub:3.141.59"
     container_name: selenium-hub

app/main.py

@@ -1,7 +1,6 @@
 from scraper import scrap
 import os
 import json
-import time
 
 urls = ["https://digitalprojects.wpi.art/monet/artworks"]
 hrefs = []

scripts/commands.py

@@ -0,0 +1,33 @@
+import subprocess
+
+
+def quitCondition(command: str) -> bool:
+    return command in ["q", "quit", "exit", "stop"]
+
+
+def helpCondition(command: str) -> bool:
+    return command in ["h", "help"]
+
+
+def clearCondition(command: str) -> bool:
+    return command in ["c", "clear", "cls"]
+
+
+def systemCommand(command: str) -> str:
+    words = command[slice(1, len(command))].split()
+    if words[0] == "":
+        return "Command not found. Write 'h' for help."
+    return subprocess.run(
+        f'docker exec -it webscraper {" ".join(words)}',
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    ).stdout.decode()
+
+
+def restartCondition(command: str) -> bool:
+    return command in ["r", "restart"]
+
+
+def runCondition(command: str) -> bool:
+    return command in ["run"]

scripts/start.py

@@ -0,0 +1,25 @@
+import subprocess
+
+
+def run_command(command: str) -> str:
+    process = subprocess.run(
+        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    )
+    if process.returncode != 0:
+        print(f"Error running command: {command}")
+        return process.stderr.decode()
+    return process.stdout.decode()
+
+
+def main():
+    print("Starting Docker Compose services...\n")
+    run_command("docker compose -f ../app/docker-compose.yaml up -d")
+
+    print(run_command("docker exec -it webscraper python main.py"))
+
+    print("Stopping and removing Docker Compose services...")
+    run_command("docker compose -f ../app/docker-compose.yaml  down")
+
+
+if __name__ == "__main__":
+    main()

scripts/threads/prompt.py

@@ -0,0 +1,67 @@
+import commands
+import subprocess
+import sys
+import threading
+
+
+def prompt():
+    while True:
+        command = input("> ")
+        if commands.quitCondition(command):
+            print("Stopping and removing Docker Compose services...")
+            subprocess.run(
+                "docker compose -f ../app/docker-compose.yaml down",
+                shell=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            break
+        if commands.helpCondition(command):
+            print(
+                """
+["h", "help"], - for help.
+["q", "quit", "exit", "stop"], - to stop program.
+["c", "clear", "cls"], - to clear console.
+["r", "restart"], - to restart Docker Compose services.
+["run"], - to run main.py in docker container.
+["$..."], - to evaluate command in docker container.
+"""
+            )
+            continue
+        if commands.clearCondition(command):
+            print("\n" * 100)
+            continue
+        if command.startswith("$"):
+            print(commands.systemCommand(command))
+            continue
+        if commands.restartCondition(command):
+            print("Restarting Docker Compose services...")
+            subprocess.run(
+                "docker compose -f ../app/docker-compose.yaml down",
+                shell=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            subprocess.run(
+                "docker compose -f ../app/docker-compose.yaml up -d",
+                shell=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            print("Composed!")
+            continue
+        if commands.runCondition(command):
+            print("Running main.py...")
+            print(
+                subprocess.run(
+                    "docker exec -it webscraper python main.py",
+                    shell=True,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                ).stdout.decode()
+            )
+            continue
+        else:
+            print("Command not found. Write 'h' for help.")
+            continue
+    sys.exit(0)

scripts/watch.py

@@ -0,0 +1,59 @@
+import subprocess
+import time
+import os
+import threading
+from threads.prompt import prompt
+
+
+def run_command(command: str) -> str:
+    process = subprocess.run(
+        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    )
+    if process.returncode != 0:
+        print(f"Error running command: {command}")
+        return process.stderr.decode()
+    return process.stdout.decode()
+
+
+thread = threading.Thread(target=prompt)
+
+
+def main():
+    print("Starting Docker Compose services...")
+    run_command("docker compose -f ../app/docker-compose.yaml up -d")
+    print("Composed!\n")
+    print("Running main.py...")
+    print(run_command("docker exec -it webscraper python main.py"))
+    print(
+        "\n\nWrite 'q' to stop program. Don't stop with 'Ctrl + C' otherwise docker container will be still on."
+    )
+    print("For help write 'h'.")
+    print("\nWatching for changes...")
+    thread.start()
+
+    path_to_watch = "/home/paprykdev/uni/webscraper/app"
+    before = {
+        f: os.stat(os.path.join(path_to_watch, f)).st_mtime
+        for f in os.listdir(path_to_watch)
+        if f.endswith(".py")
+    }
+
+    while True:
+        if threading.active_count() == 1:
+            break
+        time.sleep(1)
+        after = {
+            f: os.stat(os.path.join(path_to_watch, f)).st_mtime
+            for f in os.listdir(path_to_watch)
+            if f.endswith(".py")
+        }
+        for f in before:
+            if before[f] != after[f]:
+                print(f"\nDetected change in {f}")
+                print("Running main.py...")
+                print(run_command("docker exec -it webscraper python main.py"))
+                before[f] = after[f]
+
+
+if __name__ == "__main__":
+    main()

start.py

@@ -1,47 +0,0 @@
-import subprocess
-import time
-
-
-def run_command(command):
-    result = subprocess.run(
-        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-    )
-    if result.returncode != 0:
-        print(f"Error running command: {command}")
-        print(result.stderr.decode())
-    return result.stdout.decode()
-
-
-def wait_for_webscraper():
-    while True:
-        result = run_command("docker compose ps -q webscraper")
-        container_id = result.strip()
-
-        if not container_id:
-            print("Webscraper container not found.")
-            break
-
-        status = run_command(
-            f"docker inspect --format '{{.State.Status}}' {container_id}"
-        )
-
-        if status.strip() == "exited":
-            print("Webscraper has finished.")
-            break
-
-        print("Waiting for webscraper to finish...")
-        time.sleep(3)
-
-
-def main():
-    print("Starting Docker Compose services...")
-    run_command("docker compose up -d")
-
-    wait_for_webscraper()
-
-    print("Stopping and removing Docker Compose services...")
-    run_command("docker compose down")
-
-
-if __name__ == "__main__":
-    main()

start.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-docker compose up -d
-
-docker compose wait webscraper > /dev/null
-
-# docker compose down