From e8f9c38340ebf3f051db22a8dfe9dd35cbf5994c Mon Sep 17 00:00:00 2001 From: Tomasz Date: Wed, 19 Apr 2023 11:55:13 +0200 Subject: [PATCH 1/7] Monog DB test --- IUM_07/Dockerfile | 8 ++++++++ IUM_07/Jenkinsfile | 13 +++++++++++++ IUM_07/mongo_observer.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 IUM_07/Dockerfile create mode 100644 IUM_07/Jenkinsfile create mode 100644 IUM_07/mongo_observer.py diff --git a/IUM_07/Dockerfile b/IUM_07/Dockerfile new file mode 100644 index 0000000..e6b155f --- /dev/null +++ b/IUM_07/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:latest + +RUN apt update && apt install -y \ + python3-pip \ + python3 + +RUN python3 -m pip install sacred pymongo + diff --git a/IUM_07/Jenkinsfile b/IUM_07/Jenkinsfile new file mode 100644 index 0000000..1289354 --- /dev/null +++ b/IUM_07/Jenkinsfile @@ -0,0 +1,13 @@ + node { + checkout scm + //Pierwszy argument to tag, który zostania nadany zbudowanemu obrazowi + //Jeśli chcemy użyć Dockerfile z innej ścieżki niż ./Dockerfile, możemy ją podać jako drugi argument + def testImage = docker.build("sacred_pymongo", "./IUM_07/Dockerfile") + + //Wszystkie polecenia poniżej wykonają się w kontenerze, z podmontowanym Workspace Jenkinsa + testImage.inside { + dir ("IUM_07"){ + sh 'python3 mongo_observer.py' + } + } + } diff --git a/IUM_07/mongo_observer.py b/IUM_07/mongo_observer.py new file mode 100644 index 0000000..e07bdc4 --- /dev/null +++ b/IUM_07/mongo_observer.py @@ -0,0 +1,37 @@ +from sacred.observers import MongoObserver +from sacred import Experiment +import random +import time + +ex = Experiment("sacred_scopes", interactive=True) +ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017', + db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy. +# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://mongo_user:mongo_password_IUM_2021@localhost:27017 +@ex.config +def my_config(): + recipient = "Świecie" + greeting = "Witaj" + +@ex.capture +def prepare_message(recipient, greeting): + return "{0} {1}!".format(greeting, recipient) + +@ex.main +def my_main(recipient, greeting, _run): + print(prepare_message()) ## Nie musimy przekazywać wartości + counter = 0 + while counter < 20: + counter+=1 + value = counter + ms_to_wait = random.randint(5, 5000) + time.sleep(ms_to_wait/1000) + noise = 1.0 + 0.1 * (random.randint(0, 10) - 5) + # This will add an entry for training.loss metric in every second iteration. + # The resulting sequence of steps for training.loss will be 0, 2, 4, ... + if counter % 2 == 0: + _run.log_scalar("training.loss", value * 1.5 * noise, counter) + # Implicit step counter (0, 1, 2, 3, ...) + # incremented with each call for training.accuracy: + _run.log_scalar("training.accuracy", value * 2 * noise) + +ex.run() From 06eccfadf712d6802d74d95432789295bfab339c Mon Sep 17 00:00:00 2001 From: Tomasz Date: Wed, 19 Apr 2023 12:05:12 +0200 Subject: [PATCH 2/7] Fix Dockerfile path --- IUM_07/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IUM_07/Jenkinsfile b/IUM_07/Jenkinsfile index 1289354..af18b7d 100644 --- a/IUM_07/Jenkinsfile +++ b/IUM_07/Jenkinsfile @@ -2,7 +2,7 @@ checkout scm //Pierwszy argument to tag, który zostania nadany zbudowanemu obrazowi //Jeśli chcemy użyć Dockerfile z innej ścieżki niż ./Dockerfile, możemy ją podać jako drugi argument - def testImage = docker.build("sacred_pymongo", "./IUM_07/Dockerfile") + def testImage = docker.build("sacred_pymongo", "./IUM_07/") //Wszystkie polecenia poniżej wykonają się w kontenerze, z podmontowanym Workspace Jenkinsa testImage.inside { From 18c7c11acee52a8c6bf24c88a07968db416acf34 Mon Sep 17 00:00:00 2001 From: Tomasz Date: Wed, 19 Apr 2023 12:09:02 +0200 Subject: [PATCH 3/7] Add missing git dependency --- IUM_07/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/IUM_07/Dockerfile b/IUM_07/Dockerfile index e6b155f..a301308 100644 --- a/IUM_07/Dockerfile +++ b/IUM_07/Dockerfile @@ -1,6 +1,7 @@ FROM ubuntu:latest RUN apt update && apt install -y \ + git \ python3-pip \ python3 From c9bed7835daf752000999487f2186b2384f30f50 Mon Sep 17 00:00:00 2001 From: Tomasz Date: Wed, 19 Apr 2023 12:23:05 +0200 Subject: [PATCH 4/7] Poprawka loginu do bazy --- IUM_07/mongo_observer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/IUM_07/mongo_observer.py b/IUM_07/mongo_observer.py index e07bdc4..27606ed 100644 --- a/IUM_07/mongo_observer.py +++ b/IUM_07/mongo_observer.py @@ -4,8 +4,10 @@ import random import time ex = Experiment("sacred_scopes", interactive=True) -ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017', - db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy. + +ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017', db_name='sacred')) +#ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017', + # db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy. # W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://mongo_user:mongo_password_IUM_2021@localhost:27017 @ex.config def my_config(): From a022ba0b64836e06089a65181d04d6c2ec34341d Mon Sep 17 00:00:00 2001 From: Tomasz Date: Wed, 19 Apr 2023 12:54:45 +0200 Subject: [PATCH 5/7] Fix mongodb url in comments --- IUM_07.Sacred.ipynb | 128 +++++--------------------------------------- 1 file changed, 14 insertions(+), 114 deletions(-) diff --git a/IUM_07.Sacred.ipynb b/IUM_07.Sacred.ipynb index 0f41bc6..53e45f1 100644 --- a/IUM_07.Sacred.ipynb +++ b/IUM_07.Sacred.ipynb @@ -1431,7 +1431,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 2, "metadata": { "slideshow": { "slide_type": "slide" @@ -1443,10 +1443,13 @@ "output_type": "stream", "text": [ "Collecting pymongo\n", - " Downloading https://files.pythonhosted.org/packages/10/3b/46541b4ee3000019b8ef5b1847292ddc77f492c162bc4d49c424db7fc97a/pymongo-4.1.1-cp36-cp36m-manylinux1_x86_64.whl (464kB)\n", - "\u001b[K 100% |████████████████████████████████| 471kB 959kB/s ta 0:00:01\n", - "\u001b[?25hInstalling collected packages: pymongo\n", - "Successfully installed pymongo-4.1.1\n" + " Downloading pymongo-4.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (492 kB)\n", + "\u001b[K |████████████████████████████████| 492 kB 1.2 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting dnspython<3.0.0,>=1.16.0\n", + " Downloading dnspython-2.3.0-py3-none-any.whl (283 kB)\n", + "\u001b[K |████████████████████████████████| 283 kB 1.0 MB/s eta 0:00:01\n", + "\u001b[?25hInstalling collected packages: dnspython, pymongo\n", + "Successfully installed dnspython-2.3.0 pymongo-4.3.3\n" ] } ], @@ -1456,7 +1459,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": { "slideshow": { "slide_type": "slide" @@ -1467,77 +1470,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO - sacred_scopes - Running command 'my_main'\n", - "ERROR - sacred_scopes - Failed after 0:00:30!\n", - "ERROR - sacred_scopes - Traceback (most recent call last):\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/run.py\", line 235, in __call__\n", - " self._emit_started()\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/run.py\", line 333, in _emit_started\n", - " _id=self._id,\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\", line 258, in started_event\n", - " self.insert()\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\", line 367, in insert\n", - " c.next()[\"_id\"] + 1 if self.runs.count_documents({}, limit=1) else 1\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/collection.py\", line 1811, in count_documents\n", - " return self._retryable_non_cursor_read(_cmd, session)\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/collection.py\", line 1816, in _retryable_non_cursor_read\n", - " with client._tmp_session(session) as s:\n", - " File \"/usr/lib/python3.6/contextlib.py\", line 81, in __enter__\n", - " return next(self.gen)\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\", line 1676, in _tmp_session\n", - " s = self._ensure_session(session)\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\", line 1663, in _ensure_session\n", - " return self.__start_session(True, causal_consistency=False)\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\", line 1608, in __start_session\n", - " self._topology._check_implicit_session_support()\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/topology.py\", line 519, in _check_implicit_session_support\n", - " self._check_session_support()\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/topology.py\", line 536, in _check_session_support\n", - " readable_server_selector, self._settings.server_selection_timeout, None\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/topology.py\", line 229, in _select_servers_loop\n", - " % (self._error_message(selector), timeout, self.description)\n", - "pymongo.errors.ServerSelectionTimeoutError: localhost:27017: [Errno 111] Connection refused, Timeout: 30s, Topology Description: ]>\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/run.py\", line 429, in _final_call\n", - " getattr(observer, method)(**kwargs)\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\", line 283, in failed_event\n", - " self.final_save(attempts=1)\n", - " File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\", line 421, in final_save\n", - " os.makedirs(self.failure_dir, exist_ok=True)\n", - " File \"/usr/lib/python3.6/os.py\", line 205, in makedirs\n", - " head, tail = path.split(name)\n", - " File \"/usr/lib/python3.6/posixpath.py\", line 107, in split\n", - " p = os.fspath(p)\n", - "TypeError: expected str, bytes or os.PathLike object, not NoneType\n", - "\n" - ] - }, - { - "ename": "ServerSelectionTimeoutError", - "evalue": "localhost:27017: [Errno 111] Connection refused, Timeout: 30s, Topology Description: ]>", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mServerSelectionTimeoutError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprepare_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m## Nie musimy przekazywać wartości\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/sacred/experiment.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, command_name, config_updates, named_configs, info, meta_info, options)\u001b[0m\n\u001b[1;32m 274\u001b[0m \u001b[0mcommand_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig_updates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnamed_configs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta_info\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 275\u001b[0m )\n\u001b[0;32m--> 276\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 277\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/sacred/run.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 233\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 234\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mcapture_stdout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_emit_started\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 236\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_heartbeat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 237\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_execute_pre_run_hooks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/sacred/run.py\u001b[0m in \u001b[0;36m_emit_started\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 331\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 332\u001b[0m \u001b[0mmeta_info\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmeta_info\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m \u001b[0m_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 334\u001b[0m )\n\u001b[1;32m 335\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_id\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\u001b[0m in \u001b[0;36mstarted_event\u001b[0;34m(self, ex_info, command, host_info, start_time, config, meta_info, _id)\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;31m# save sources\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_entry\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"experiment\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"sources\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_sources\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mex_info\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 258\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 259\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_entry\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"_id\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\u001b[0m in \u001b[0;36minsert\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 365\u001b[0m \u001b[0mc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"_id\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpymongo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDESCENDING\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlimit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 366\u001b[0m self.run_entry[\"_id\"] = (\n\u001b[0;32m--> 367\u001b[0;31m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"_id\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mruns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcount_documents\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 368\u001b[0m )\n\u001b[1;32m 369\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/collection.py\u001b[0m in \u001b[0;36mcount_documents\u001b[0;34m(self, filter, session, comment, **kwargs)\u001b[0m\n\u001b[1;32m 1809\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"n\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1811\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_retryable_non_cursor_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_cmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1812\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1813\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_retryable_non_cursor_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/collection.py\u001b[0m in \u001b[0;36m_retryable_non_cursor_read\u001b[0;34m(self, func, session)\u001b[0m\n\u001b[1;32m 1814\u001b[0m \u001b[0;34m\"\"\"Non-cursor read helper to handle implicit session creation.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1815\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__database\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1816\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tmp_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1817\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_retryable_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_preference_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1818\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3.6/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 79\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 82\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 83\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m_tmp_session\u001b[0;34m(self, session, close)\u001b[0m\n\u001b[1;32m 1674\u001b[0m \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1676\u001b[0;31m \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ensure_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1677\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1678\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m_ensure_session\u001b[0;34m(self, session)\u001b[0m\n\u001b[1;32m 1661\u001b[0m \u001b[0;31m# Don't make implicit sessions causally consistent. Applications\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1662\u001b[0m \u001b[0;31m# should always opt-in.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1663\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__start_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcausal_consistency\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1664\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mConfigurationError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mInvalidOperation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1665\u001b[0m \u001b[0;31m# Sessions not supported.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m__start_session\u001b[0;34m(self, implicit, **kwargs)\u001b[0m\n\u001b[1;32m 1606\u001b[0m \u001b[0;31m# Raises ConfigurationError if sessions are not supported.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1607\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mimplicit\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1608\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_topology\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_implicit_session_support\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1609\u001b[0m \u001b[0mserver_session\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_EmptyServerSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1610\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/topology.py\u001b[0m in \u001b[0;36m_check_implicit_session_support\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 517\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_check_implicit_session_support\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 518\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 519\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_session_support\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 520\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 521\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_check_session_support\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/topology.py\u001b[0m in \u001b[0;36m_check_session_support\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 534\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_description\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable_servers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 535\u001b[0m self._select_servers_loop(\n\u001b[0;32m--> 536\u001b[0;31m \u001b[0mreadable_server_selector\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_settings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserver_selection_timeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 537\u001b[0m )\n\u001b[1;32m 538\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/topology.py\u001b[0m in \u001b[0;36m_select_servers_loop\u001b[0;34m(self, selector, timeout, address)\u001b[0m\n\u001b[1;32m 227\u001b[0m raise ServerSelectionTimeoutError(\n\u001b[1;32m 228\u001b[0m \u001b[0;34m\"%s, Timeout: %ss, Topology Description: %r\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 229\u001b[0;31m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_error_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselector\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdescription\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 230\u001b[0m )\n\u001b[1;32m 231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mServerSelectionTimeoutError\u001b[0m: localhost:27017: [Errno 111] Connection refused, Timeout: 30s, Topology Description: ]>" + "INFO - sacred_scopes - Running command 'my_main'\n" ] } ], @@ -1548,7 +1481,7 @@ "ex = Experiment(\"sacred_scopes\", interactive=True)\n", "ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017',\n", " db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.\n", - "# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://mongo_user:mongo_password_IUM_2021@localhost:27017\n", + "# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://admin:IUM_2021@172.17.0.1:27017\n", "@ex.config\n", "def my_config():\n", " recipient = \"Świecie\"\n", @@ -1596,46 +1529,13 @@ }, { "cell_type": "code", - "execution_count": 192, + "execution_count": null, "metadata": { "slideshow": { "slide_type": "slide" } }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO - sacred_scopes - Running command 'my_main'\n", - "INFO - sacred_scopes - Started run with ID \"9\"\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Witaj Świecie!\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO - sacred_scopes - Completed after 0:00:50\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 192, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from sacred.observers import MongoObserver\n", "from sacred import Experiment\n", @@ -1645,7 +1545,7 @@ "ex = Experiment(\"sacred_scopes\", interactive=True)\n", "ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017',\n", " db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.\n", - "# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://mongo_user:mongo_password_IUM_2021@localhost:27017\n", + "# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://admin:IUM_2021@172.17.0.1:27017\n", "@ex.config\n", "def my_config():\n", " recipient = \"Świecie\"\n", From 0d2ac2309af6c38c4bb5a35d1dd4a43b34dbc0f6 Mon Sep 17 00:00:00 2001 From: Tomasz Date: Thu, 20 Apr 2023 11:28:19 +0200 Subject: [PATCH 6/7] =?UTF-8?q?Zaktualizowaneo=20materia=C5=82y=20dot.=20M?= =?UTF-8?q?LFLow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- IUM_08.MLFlow.ipynb | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/IUM_08.MLFlow.ipynb b/IUM_08.MLFlow.ipynb index 9efe928..86ad396 100644 --- a/IUM_08.MLFlow.ipynb +++ b/IUM_08.MLFlow.ipynb @@ -8,7 +8,7 @@ } }, "source": [ - "![Logo 1](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech1.jpg)\n", + "## ![Logo 1](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech1.jpg)\n", "
\n", "

Inżynieria uczenia maszynowego

\n", "

8. MLFlow [laboratoria]

\n", @@ -73,7 +73,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": { "slideshow": { "slide_type": "slide" @@ -88,7 +88,16 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "!mkdir -p IUM_08/examples/sklearn_elasticnet_wine/" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "metadata": { "slideshow": { "slide_type": "slide" @@ -190,6 +199,7 @@ " mlflow.log_metric(\"mae\", mae)\n", " \n", " # Infer model signature to log it\n", + " # Więcej o sygnaturach: https://mlflow.org/docs/latest/models.html?highlight=signature#model-signature\n", " signature = mlflow.models.signature.infer_signature(train_x, lr.predict(train_x))\n", "\n", " tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme\n", @@ -948,7 +958,7 @@ "source": [ "# Zadania [10p pkt]\n", "1. Dodaj do swojego projektu logowanie parametrów i metryk za pomocą MLflow (polecenia `mlflow.log_param` i `mlflow.log_metric`\n", - "2. Dodaj plik MLProject definiujący polecenia do trenowania i testowania, ich parametry wywołania oraz środowisko (użyj zdefiniowanego wcześniej obrazu Docker)" + "2. Dodaj plik MLProject definiujący polecenia do trenowania i testowania, ich parametry wywołania oraz środowisko (Conda albo Docker)" ] }, { @@ -1390,7 +1400,7 @@ } }, "source": [ - "## Zadania\n", + "## Zadania (termin: 14.05 EOD) \n", "1. [2 pkt] Dodaj do joba treningowego wywołania MLflow, tak, żeby przy każdym uruchomieniu stworzyć i zarchiwizować katalog z modelem. Plik MLmodel powinien zawierać pola:\n", " - signature\n", " - input_example\n", @@ -1425,6 +1435,10 @@ " - CLI: `export MLFLOW_TRACKING_URI=http://172.17.0.1:5000`\n", " \n", "- Żeby klient MLflow działający w kontenerze docker mógł zapisywać i pdczytywać artefakty, muszą Państwo podmonotwać katalog `/tmp/mlruns` i `/mlruns` (ten drugi po to, żeby po restarcie serwera katalog nie został wyczyszczony)\n", + " - jak podmontować: ```docker.image('my-image').inside('-v /tmp/mlruns:/tmp/mlruns' -v /mlruns:/mlruns')```\n", + " - Przykład działąjącego joba na Jenkinsie zapisującego wyniki do MLFlow: https://tzietkiewicz.vm.wmi.amu.edu.pl:8081/job/gitea-test/job/ium-helloworld/job/master/\n", + " - Repo: https://git.wmi.amu.edu.pl/tzietkiewicz/ium-helloworld/src/branch/master\n", + "\n", "- Proszę ustawić nazwę eksperymentu na numer indeksu, dzięki temu każdy z Państwa będzie widział swoje eksperymenty oddzielnie:\n", "`mlflow.set_experiment(\"s123456\")`" ] @@ -1435,7 +1449,7 @@ "celltoolbar": "Slideshow", "email": "tomasz.zietkiewicz@amu.edu.pl", "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -1450,7 +1464,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "slideshow": { "slide_type": "slide" From b079dbb97847a21d4b4a8052a45c0fa03ea27683 Mon Sep 17 00:00:00 2001 From: Tomasz Date: Thu, 20 Apr 2023 12:13:19 +0200 Subject: [PATCH 7/7] Poprawki --- IUM_04.Konteneryzacja.ipynb | 2 +- IUM_07.Sacred.ipynb | 229 +++++++++++++++--------------------- IUM_08.MLFlow.ipynb | 2 +- 3 files changed, 96 insertions(+), 137 deletions(-) diff --git a/IUM_04.Konteneryzacja.ipynb b/IUM_04.Konteneryzacja.ipynb index bf14a30..bde53a7 100644 --- a/IUM_04.Konteneryzacja.ipynb +++ b/IUM_04.Konteneryzacja.ipynb @@ -248,7 +248,7 @@ } }, "source": [ - "Przykładowy Dockerfile:\n", + "### Przykładowy Dockerfile:\n", "```Dockerfile\n", "# Nasz obraz będzie dzidziczył z obrazu Ubuntu w wersji latest\n", "FROM ubuntu:latest\n", diff --git a/IUM_07.Sacred.ipynb b/IUM_07.Sacred.ipynb index 53e45f1..c3db448 100644 --- a/IUM_07.Sacred.ipynb +++ b/IUM_07.Sacred.ipynb @@ -115,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": { "slideshow": { "slide_type": "slide" @@ -126,40 +126,18 @@ "name": "stdout", "output_type": "stream", "text": [ - "Collecting sacred\n", - " Downloading sacred-0.8.4-py2.py3-none-any.whl (107 kB)\n", - "\u001b[K |████████████████████████████████| 107 kB 637 kB/s eta 0:00:01\n", - "\u001b[?25hCollecting py-cpuinfo>=4.0\n", - " Downloading py_cpuinfo-9.0.0-py3-none-any.whl (22 kB)\n", - "Collecting GitPython\n", - " Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\n", - "\u001b[K |████████████████████████████████| 184 kB 617 kB/s eta 0:00:01\n", - "\u001b[?25hCollecting munch<3.0,>=2.5\n", - " Downloading munch-2.5.0-py2.py3-none-any.whl (10 kB)\n", - "Collecting wrapt<2.0,>=1.0\n", - " Downloading wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (78 kB)\n", - "\u001b[K |████████████████████████████████| 78 kB 508 kB/s eta 0:00:01\n", - "\u001b[?25hCollecting colorama>=0.4\n", - " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", - "Collecting docopt<1.0,>=0.3\n", - " Downloading docopt-0.6.2.tar.gz (25 kB)\n", - "Collecting jsonpickle>=1.2\n", - " Downloading jsonpickle-3.0.1-py2.py3-none-any.whl (40 kB)\n", - "\u001b[K |████████████████████████████████| 40 kB 518 kB/s eta 0:00:01\n", - "\u001b[?25hRequirement already satisfied: packaging>=18.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (23.0)\n", + "Requirement already satisfied: sacred in /home/tomek/miniconda3/lib/python3.9/site-packages (0.8.4)\n", + "Requirement already satisfied: jsonpickle>=1.2 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (3.0.1)\n", + "Requirement already satisfied: colorama>=0.4 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (0.4.6)\n", + "Requirement already satisfied: GitPython in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (3.1.31)\n", + "Requirement already satisfied: py-cpuinfo>=4.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (9.0.0)\n", + "Requirement already satisfied: wrapt<2.0,>=1.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (1.15.0)\n", + "Requirement already satisfied: munch<3.0,>=2.5 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (2.5.0)\n", + "Requirement already satisfied: docopt<1.0,>=0.3 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (0.6.2)\n", + "Requirement already satisfied: packaging>=18.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (23.0)\n", "Requirement already satisfied: six in /home/tomek/miniconda3/lib/python3.9/site-packages (from munch<3.0,>=2.5->sacred) (1.16.0)\n", - "Collecting gitdb<5,>=4.0.1\n", - " Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n", - "\u001b[K |████████████████████████████████| 62 kB 430 kB/s eta 0:00:01\n", - "\u001b[?25hCollecting smmap<6,>=3.0.1\n", - " Using cached smmap-5.0.0-py3-none-any.whl (24 kB)\n", - "Building wheels for collected packages: docopt\n", - " Building wheel for docopt (setup.py) ... \u001b[?25ldone\n", - "\u001b[?25h Created wheel for docopt: filename=docopt-0.6.2-py2.py3-none-any.whl size=13723 sha256=9ae6354916cc7817db186faf89036b67a6562e0391c04e901e72698d9ce5bd8b\n", - " Stored in directory: /home/tomek/.cache/pip/wheels/70/4a/46/1309fc853b8d395e60bafaf1b6df7845bdd82c95fd59dd8d2b\n", - "Successfully built docopt\n", - "Installing collected packages: smmap, gitdb, wrapt, py-cpuinfo, munch, jsonpickle, GitPython, docopt, colorama, sacred\n", - "Successfully installed GitPython-3.1.31 colorama-0.4.6 docopt-0.6.2 gitdb-4.0.10 jsonpickle-3.0.1 munch-2.5.0 py-cpuinfo-9.0.0 sacred-0.8.4 smmap-5.0.0 wrapt-1.15.0\n" + "Requirement already satisfied: gitdb<5,>=4.0.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from GitPython->sacred) (4.0.10)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from gitdb<5,>=4.0.1->GitPython->sacred) (5.0.0)\n" ] } ], @@ -208,7 +186,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 5, "metadata": { "slideshow": { "slide_type": "fragment" @@ -262,7 +240,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 6, "metadata": { "slideshow": { "slide_type": "fragment" @@ -300,6 +278,7 @@ " -f --force Disable warnings about suspicious changes for\r\n", " this run.\r\n", " -h --help Print this help message and exit.\r\n", + " -i VALUE --id=VALUE Set the id for this run.\r\n", " -l VALUE --loglevel=VALUE Set the LogLevel. Loglevel either as 0 - 50 or\r\n", " as string: DEBUG(10), INFO(20), WARNING(30),\r\n", " ERROR(40), CRITICAL(50)\r\n", @@ -378,7 +357,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": { "slideshow": { "slide_type": "slide" @@ -407,7 +386,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "metadata": { "slideshow": { "slide_type": "fragment" @@ -434,10 +413,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 6, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -459,7 +438,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 10, "metadata": { "slideshow": { "slide_type": "fragment" @@ -472,7 +451,7 @@ "{'recipient': 'Świecie', 'greeting': 'Witaj', 'message': 'Witaj Świecie!'}" ] }, - "execution_count": 19, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -521,7 +500,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 14, "metadata": { "slideshow": { "slide_type": "fragment" @@ -546,7 +525,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 15, "metadata": { "slideshow": { "slide_type": "fragment" @@ -563,7 +542,7 @@ " greeting = 'Witaj'\r\n", " message = 'Witaj Świecie!'\r\n", " recipient = 'Świecie'\r\n", - " seed = 539001265 \u001b[2m# the random seed for this experiment\u001b[0m\r\n", + " seed = 269258424 \u001b[2m# the random seed for this experiment\u001b[0m\r\n", "INFO - sacred_scopes - Completed after 0:00:00\r\n" ] } @@ -574,7 +553,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 16, "metadata": { "slideshow": { "slide_type": "fragment" @@ -591,7 +570,7 @@ " greeting = 'Witaj'\r\n", " message = 'Witaj Przygodo!'\r\n", "\u001b[34m recipient = 'Przygodo'\u001b[0m\r\n", - " seed = 215765170 \u001b[2m# the random seed for this experiment\u001b[0m\r\n", + " seed = 667939214 \u001b[2m# the random seed for this experiment\u001b[0m\r\n", "INFO - sacred_scopes - Completed after 0:00:00\r\n" ] } @@ -613,24 +592,13 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "slideshow": { "slide_type": "fragment" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'recipient': 'samotności', 'greeting': 'Żegnaj'}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# %load IUM_07/config.json\n", "{\n", @@ -641,7 +609,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 20, "metadata": { "slideshow": { "slide_type": "slide" @@ -671,7 +639,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 21, "metadata": { "slideshow": { "slide_type": "fragment" @@ -702,7 +670,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 22, "metadata": { "slideshow": { "slide_type": "fragment" @@ -712,10 +680,10 @@ { "data": { "text/plain": [ - "{'recipient': 'samotności', 'greeting': 'Żegnaj', 'seed': 529757761}" + "{'recipient': 'samotności', 'greeting': 'Żegnaj', 'seed': 877272352}" ] }, - "execution_count": 121, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -737,7 +705,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 23, "metadata": { "slideshow": { "slide_type": "fragment" @@ -783,7 +751,7 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 24, "metadata": { "slideshow": { "slide_type": "slide" @@ -811,10 +779,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 193, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -864,11 +832,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 20\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 1\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 2\r\n", + "drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 3\r\n", + "drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 _resources\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 _sources\r\n" + ] + } + ], "source": [ - "!ls -l runs" + "!ls -l my_runs" ] }, { @@ -906,21 +887,13 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "slideshow": { "slide_type": "slide" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting IUM_07/file_observer.py\n" - ] - } - ], + "outputs": [], "source": [ "%%writefile IUM_07/file_observer.py\n", "from sacred.observers import FileStorageObserver\n", @@ -946,7 +919,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 29, "metadata": { "slideshow": { "slide_type": "fragment" @@ -958,7 +931,7 @@ "output_type": "stream", "text": [ "INFO - file_observer - Running command 'my_main'\r\n", - "INFO - file_observer - Started run with ID \"1\"\r\n", + "INFO - file_observer - Started run with ID \"4\"\r\n", "Witaj Świecie!\r\n", "INFO - file_observer - Completed after 0:00:00\r\n" ] @@ -981,7 +954,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 30, "metadata": { "slideshow": { "slide_type": "slide" @@ -992,9 +965,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "total 0\r\n", - "drwxr-xr-x 1 tomek tomek 512 Apr 25 09:51 1\r\n", - "drwxr-xr-x 1 tomek tomek 512 Apr 25 09:51 _sources\r\n" + "total 24\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 1\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 2\r\n", + "drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 3\r\n", + "drwxr-sr-x 2 tomek tomek 4096 Apr 20 12:09 4\r\n", + "drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 _resources\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 _sources\r\n" ] } ], @@ -1004,25 +981,13 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "slideshow": { "slide_type": "slide" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "total 4\r\n", - "-rw-r--r-- 1 tomek tomek 77 Apr 25 09:51 config.json\r\n", - "-rw-r--r-- 1 tomek tomek 159 Apr 25 09:51 cout.txt\r\n", - "-rw-r--r-- 1 tomek tomek 2 Apr 25 09:51 metrics.json\r\n", - "-rw-r--r-- 1 tomek tomek 1659 Apr 25 09:51 run.json\r\n" - ] - } - ], + "outputs": [], "source": [ "!ls -l my_runs/1" ] @@ -1047,7 +1012,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 31, "metadata": { "slideshow": { "slide_type": "slide" @@ -1153,7 +1118,7 @@ }, { "cell_type": "code", - "execution_count": 170, + "execution_count": 32, "metadata": { "slideshow": { "slide_type": "slide" @@ -1165,7 +1130,7 @@ "output_type": "stream", "text": [ "total 4\r\n", - "-rw-rw-r-- 1 tomek tomek 463 kwi 26 10:21 file_observer_bb0a5c4720d1072b641d23da080696b6.py\r\n" + "-rw-r--r-- 1 tomek tomek 464 May 28 2022 file_observer_cd34a0ef4a32fb0a966eaa01ea6371ad.py\r\n" ] } ], @@ -1184,8 +1149,7 @@ }, "outputs": [], "source": [ - "## Źródła zostały zapisane\n", - "# %load my_runs/_sources/file_observer_bb0a5c4720d1072b641d23da080696b6.py\n", + "# %load my_runs/_sources/file_observer_cd34a0ef4a32fb0a966eaa01ea6371ad.py\n", "from sacred.observers import FileStorageObserver\n", "from sacred import Experiment\n", "\n", @@ -1204,7 +1168,7 @@ "\n", "@ex.automain\n", "def my_main(recipient, greeting):\n", - " print(prepare_message()) ## Nie musimy przekazywać wartości" + " print(prepare_message()) ## Nie musimy przekazywać wartości\n" ] }, { @@ -1220,7 +1184,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 36, "metadata": { "slideshow": { "slide_type": "slide" @@ -1232,7 +1196,7 @@ "output_type": "stream", "text": [ "INFO - file_observer - Running command 'my_main'\n", - "INFO - file_observer - Started run with ID \"2\"\n", + "INFO - file_observer - Started run with ID \"5\"\n", "INFO - file_observer - Completed after 0:00:00\n" ] }, @@ -1275,7 +1239,7 @@ }, { "cell_type": "code", - "execution_count": 185, + "execution_count": 37, "metadata": { "slideshow": { "slide_type": "slide" @@ -1287,13 +1251,13 @@ "output_type": "stream", "text": [ "{\r\n", - " \"prepare_message_ts\": \"2021-04-26 10:39:59.268539\"\r\n", + " \"prepare_message_ts\": \"2023-04-20 12:10:28.197315\"\r\n", "}" ] } ], "source": [ - "cat my_runs/6/info.json" + "cat my_runs/5/info.json" ] }, { @@ -1326,7 +1290,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 38, "metadata": { "slideshow": { "slide_type": "slide" @@ -1338,7 +1302,7 @@ "output_type": "stream", "text": [ "INFO - resources - Running command 'my_main'\n", - "INFO - resources - Started run with ID \"3\"\n", + "INFO - resources - Started run with ID \"6\"\n", "INFO - resources - Completed after 0:00:00\n" ] }, @@ -1353,10 +1317,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 6, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -1379,7 +1343,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -1397,21 +1361,21 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 41, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "my_runs/3/run.json: \"name\": \"resources\",\r\n", - "my_runs/3/run.json: \"resources\": [\r\n", - "my_runs/3/run.json: \"my_runs/_resources/Iris_717820ef0af287ff346c5cabfb4c612c.csv\"\r\n" + "my_runs/6/run.json: \"name\": \"resources\",\r\n", + "my_runs/6/run.json: \"resources\": [\r\n", + "my_runs/6/run.json: \"my_runs/_resources/Iris_717820ef0af287ff346c5cabfb4c612c.csv\"\r\n" ] } ], "source": [ - "!grep -e \"resources\" -R my_runs/3" + "!grep -e \"resources\" -R my_runs/6" ] }, { @@ -1431,7 +1395,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 42, "metadata": { "slideshow": { "slide_type": "slide" @@ -1442,14 +1406,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Collecting pymongo\n", - " Downloading pymongo-4.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (492 kB)\n", - "\u001b[K |████████████████████████████████| 492 kB 1.2 MB/s eta 0:00:01\n", - "\u001b[?25hCollecting dnspython<3.0.0,>=1.16.0\n", - " Downloading dnspython-2.3.0-py3-none-any.whl (283 kB)\n", - "\u001b[K |████████████████████████████████| 283 kB 1.0 MB/s eta 0:00:01\n", - "\u001b[?25hInstalling collected packages: dnspython, pymongo\n", - "Successfully installed dnspython-2.3.0 pymongo-4.3.3\n" + "Requirement already satisfied: pymongo in /home/tomek/miniconda3/lib/python3.9/site-packages (4.3.3)\r\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from pymongo) (2.3.0)\r\n" ] } ], @@ -1470,7 +1428,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO - sacred_scopes - Running command 'my_main'\n" + "INFO - sacred_scopes - Running command 'my_main'\n", + "ERROR - sacred_scopes - Failed after 0:00:30!\n" ] } ], @@ -1479,7 +1438,7 @@ "from sacred import Experiment\n", "\n", "ex = Experiment(\"sacred_scopes\", interactive=True)\n", - "ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017',\n", + "ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017',\n", " db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.\n", "# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://admin:IUM_2021@172.17.0.1:27017\n", "@ex.config\n", @@ -1543,7 +1502,7 @@ "import time\n", "\n", "ex = Experiment(\"sacred_scopes\", interactive=True)\n", - "ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017',\n", + "ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017',\n", " db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.\n", "# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://admin:IUM_2021@172.17.0.1:27017\n", "@ex.config\n", diff --git a/IUM_08.MLFlow.ipynb b/IUM_08.MLFlow.ipynb index 86ad396..1c5e669 100644 --- a/IUM_08.MLFlow.ipynb +++ b/IUM_08.MLFlow.ipynb @@ -12,7 +12,7 @@ "
\n", "

Inżynieria uczenia maszynowego

\n", "

8. MLFlow [laboratoria]

\n", - "

Tomasz Ziętkiewicz (2022)

\n", + "

Tomasz Ziętkiewicz (2023)

\n", "
\n", "\n", "![Logo 2](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech2.jpg)"