diff --git a/IUM_04.Konteneryzacja.ipynb b/IUM_04.Konteneryzacja.ipynb index bf14a30..bde53a7 100644 --- a/IUM_04.Konteneryzacja.ipynb +++ b/IUM_04.Konteneryzacja.ipynb @@ -248,7 +248,7 @@ } }, "source": [ - "Przykładowy Dockerfile:\n", + "### Przykładowy Dockerfile:\n", "```Dockerfile\n", "# Nasz obraz będzie dzidziczył z obrazu Ubuntu w wersji latest\n", "FROM ubuntu:latest\n", diff --git a/IUM_07.Sacred.ipynb b/IUM_07.Sacred.ipynb index 53e45f1..c3db448 100644 --- a/IUM_07.Sacred.ipynb +++ b/IUM_07.Sacred.ipynb @@ -115,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": { "slideshow": { "slide_type": "slide" @@ -126,40 +126,18 @@ "name": "stdout", "output_type": "stream", "text": [ - "Collecting sacred\n", - " Downloading sacred-0.8.4-py2.py3-none-any.whl (107 kB)\n", - "\u001b[K |████████████████████████████████| 107 kB 637 kB/s eta 0:00:01\n", - "\u001b[?25hCollecting py-cpuinfo>=4.0\n", - " Downloading py_cpuinfo-9.0.0-py3-none-any.whl (22 kB)\n", - "Collecting GitPython\n", - " Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\n", - "\u001b[K |████████████████████████████████| 184 kB 617 kB/s eta 0:00:01\n", - "\u001b[?25hCollecting munch<3.0,>=2.5\n", - " Downloading munch-2.5.0-py2.py3-none-any.whl (10 kB)\n", - "Collecting wrapt<2.0,>=1.0\n", - " Downloading wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (78 kB)\n", - "\u001b[K |████████████████████████████████| 78 kB 508 kB/s eta 0:00:01\n", - "\u001b[?25hCollecting colorama>=0.4\n", - " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", - "Collecting docopt<1.0,>=0.3\n", - " Downloading docopt-0.6.2.tar.gz (25 kB)\n", - "Collecting jsonpickle>=1.2\n", - " Downloading jsonpickle-3.0.1-py2.py3-none-any.whl (40 kB)\n", - "\u001b[K |████████████████████████████████| 40 kB 518 kB/s eta 0:00:01\n", - "\u001b[?25hRequirement already satisfied: packaging>=18.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (23.0)\n", + "Requirement already satisfied: sacred in /home/tomek/miniconda3/lib/python3.9/site-packages (0.8.4)\n", + "Requirement already satisfied: jsonpickle>=1.2 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (3.0.1)\n", + "Requirement already satisfied: colorama>=0.4 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (0.4.6)\n", + "Requirement already satisfied: GitPython in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (3.1.31)\n", + "Requirement already satisfied: py-cpuinfo>=4.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (9.0.0)\n", + "Requirement already satisfied: wrapt<2.0,>=1.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (1.15.0)\n", + "Requirement already satisfied: munch<3.0,>=2.5 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (2.5.0)\n", + "Requirement already satisfied: docopt<1.0,>=0.3 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (0.6.2)\n", + "Requirement already satisfied: packaging>=18.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (23.0)\n", "Requirement already satisfied: six in /home/tomek/miniconda3/lib/python3.9/site-packages (from munch<3.0,>=2.5->sacred) (1.16.0)\n", - "Collecting gitdb<5,>=4.0.1\n", - " Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n", - "\u001b[K |████████████████████████████████| 62 kB 430 kB/s eta 0:00:01\n", - "\u001b[?25hCollecting smmap<6,>=3.0.1\n", - " Using cached smmap-5.0.0-py3-none-any.whl (24 kB)\n", - "Building wheels for collected packages: docopt\n", - " Building wheel for docopt (setup.py) ... \u001b[?25ldone\n", - "\u001b[?25h Created wheel for docopt: filename=docopt-0.6.2-py2.py3-none-any.whl size=13723 sha256=9ae6354916cc7817db186faf89036b67a6562e0391c04e901e72698d9ce5bd8b\n", - " Stored in directory: /home/tomek/.cache/pip/wheels/70/4a/46/1309fc853b8d395e60bafaf1b6df7845bdd82c95fd59dd8d2b\n", - "Successfully built docopt\n", - "Installing collected packages: smmap, gitdb, wrapt, py-cpuinfo, munch, jsonpickle, GitPython, docopt, colorama, sacred\n", - "Successfully installed GitPython-3.1.31 colorama-0.4.6 docopt-0.6.2 gitdb-4.0.10 jsonpickle-3.0.1 munch-2.5.0 py-cpuinfo-9.0.0 sacred-0.8.4 smmap-5.0.0 wrapt-1.15.0\n" + "Requirement already satisfied: gitdb<5,>=4.0.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from GitPython->sacred) (4.0.10)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from gitdb<5,>=4.0.1->GitPython->sacred) (5.0.0)\n" ] } ], @@ -208,7 +186,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 5, "metadata": { "slideshow": { "slide_type": "fragment" @@ -262,7 +240,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 6, "metadata": { "slideshow": { "slide_type": "fragment" @@ -300,6 +278,7 @@ " -f --force Disable warnings about suspicious changes for\r\n", " this run.\r\n", " -h --help Print this help message and exit.\r\n", + " -i VALUE --id=VALUE Set the id for this run.\r\n", " -l VALUE --loglevel=VALUE Set the LogLevel. Loglevel either as 0 - 50 or\r\n", " as string: DEBUG(10), INFO(20), WARNING(30),\r\n", " ERROR(40), CRITICAL(50)\r\n", @@ -378,7 +357,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": { "slideshow": { "slide_type": "slide" @@ -407,7 +386,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "metadata": { "slideshow": { "slide_type": "fragment" @@ -434,10 +413,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 6, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -459,7 +438,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 10, "metadata": { "slideshow": { "slide_type": "fragment" @@ -472,7 +451,7 @@ "{'recipient': 'Świecie', 'greeting': 'Witaj', 'message': 'Witaj Świecie!'}" ] }, - "execution_count": 19, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -521,7 +500,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 14, "metadata": { "slideshow": { "slide_type": "fragment" @@ -546,7 +525,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 15, "metadata": { "slideshow": { "slide_type": "fragment" @@ -563,7 +542,7 @@ " greeting = 'Witaj'\r\n", " message = 'Witaj Świecie!'\r\n", " recipient = 'Świecie'\r\n", - " seed = 539001265 \u001b[2m# the random seed for this experiment\u001b[0m\r\n", + " seed = 269258424 \u001b[2m# the random seed for this experiment\u001b[0m\r\n", "INFO - sacred_scopes - Completed after 0:00:00\r\n" ] } @@ -574,7 +553,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 16, "metadata": { "slideshow": { "slide_type": "fragment" @@ -591,7 +570,7 @@ " greeting = 'Witaj'\r\n", " message = 'Witaj Przygodo!'\r\n", "\u001b[34m recipient = 'Przygodo'\u001b[0m\r\n", - " seed = 215765170 \u001b[2m# the random seed for this experiment\u001b[0m\r\n", + " seed = 667939214 \u001b[2m# the random seed for this experiment\u001b[0m\r\n", "INFO - sacred_scopes - Completed after 0:00:00\r\n" ] } @@ -613,24 +592,13 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "slideshow": { "slide_type": "fragment" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'recipient': 'samotności', 'greeting': 'Żegnaj'}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# %load IUM_07/config.json\n", "{\n", @@ -641,7 +609,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 20, "metadata": { "slideshow": { "slide_type": "slide" @@ -671,7 +639,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 21, "metadata": { "slideshow": { "slide_type": "fragment" @@ -702,7 +670,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 22, "metadata": { "slideshow": { "slide_type": "fragment" @@ -712,10 +680,10 @@ { "data": { "text/plain": [ - "{'recipient': 'samotności', 'greeting': 'Żegnaj', 'seed': 529757761}" + "{'recipient': 'samotności', 'greeting': 'Żegnaj', 'seed': 877272352}" ] }, - "execution_count": 121, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -737,7 +705,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 23, "metadata": { "slideshow": { "slide_type": "fragment" @@ -783,7 +751,7 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 24, "metadata": { "slideshow": { "slide_type": "slide" @@ -811,10 +779,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 193, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -864,11 +832,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 20\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 1\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 2\r\n", + "drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 3\r\n", + "drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 _resources\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 _sources\r\n" + ] + } + ], "source": [ - "!ls -l runs" + "!ls -l my_runs" ] }, { @@ -906,21 +887,13 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "slideshow": { "slide_type": "slide" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting IUM_07/file_observer.py\n" - ] - } - ], + "outputs": [], "source": [ "%%writefile IUM_07/file_observer.py\n", "from sacred.observers import FileStorageObserver\n", @@ -946,7 +919,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 29, "metadata": { "slideshow": { "slide_type": "fragment" @@ -958,7 +931,7 @@ "output_type": "stream", "text": [ "INFO - file_observer - Running command 'my_main'\r\n", - "INFO - file_observer - Started run with ID \"1\"\r\n", + "INFO - file_observer - Started run with ID \"4\"\r\n", "Witaj Świecie!\r\n", "INFO - file_observer - Completed after 0:00:00\r\n" ] @@ -981,7 +954,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 30, "metadata": { "slideshow": { "slide_type": "slide" @@ -992,9 +965,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "total 0\r\n", - "drwxr-xr-x 1 tomek tomek 512 Apr 25 09:51 1\r\n", - "drwxr-xr-x 1 tomek tomek 512 Apr 25 09:51 _sources\r\n" + "total 24\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 1\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 2\r\n", + "drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 3\r\n", + "drwxr-sr-x 2 tomek tomek 4096 Apr 20 12:09 4\r\n", + "drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 _resources\r\n", + "drwsrwsr-t 2 tomek tomek 4096 May 28 2022 _sources\r\n" ] } ], @@ -1004,25 +981,13 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "slideshow": { "slide_type": "slide" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "total 4\r\n", - "-rw-r--r-- 1 tomek tomek 77 Apr 25 09:51 config.json\r\n", - "-rw-r--r-- 1 tomek tomek 159 Apr 25 09:51 cout.txt\r\n", - "-rw-r--r-- 1 tomek tomek 2 Apr 25 09:51 metrics.json\r\n", - "-rw-r--r-- 1 tomek tomek 1659 Apr 25 09:51 run.json\r\n" - ] - } - ], + "outputs": [], "source": [ "!ls -l my_runs/1" ] @@ -1047,7 +1012,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 31, "metadata": { "slideshow": { "slide_type": "slide" @@ -1153,7 +1118,7 @@ }, { "cell_type": "code", - "execution_count": 170, + "execution_count": 32, "metadata": { "slideshow": { "slide_type": "slide" @@ -1165,7 +1130,7 @@ "output_type": "stream", "text": [ "total 4\r\n", - "-rw-rw-r-- 1 tomek tomek 463 kwi 26 10:21 file_observer_bb0a5c4720d1072b641d23da080696b6.py\r\n" + "-rw-r--r-- 1 tomek tomek 464 May 28 2022 file_observer_cd34a0ef4a32fb0a966eaa01ea6371ad.py\r\n" ] } ], @@ -1184,8 +1149,7 @@ }, "outputs": [], "source": [ - "## Źródła zostały zapisane\n", - "# %load my_runs/_sources/file_observer_bb0a5c4720d1072b641d23da080696b6.py\n", + "# %load my_runs/_sources/file_observer_cd34a0ef4a32fb0a966eaa01ea6371ad.py\n", "from sacred.observers import FileStorageObserver\n", "from sacred import Experiment\n", "\n", @@ -1204,7 +1168,7 @@ "\n", "@ex.automain\n", "def my_main(recipient, greeting):\n", - " print(prepare_message()) ## Nie musimy przekazywać wartości" + " print(prepare_message()) ## Nie musimy przekazywać wartości\n" ] }, { @@ -1220,7 +1184,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 36, "metadata": { "slideshow": { "slide_type": "slide" @@ -1232,7 +1196,7 @@ "output_type": "stream", "text": [ "INFO - file_observer - Running command 'my_main'\n", - "INFO - file_observer - Started run with ID \"2\"\n", + "INFO - file_observer - Started run with ID \"5\"\n", "INFO - file_observer - Completed after 0:00:00\n" ] }, @@ -1275,7 +1239,7 @@ }, { "cell_type": "code", - "execution_count": 185, + "execution_count": 37, "metadata": { "slideshow": { "slide_type": "slide" @@ -1287,13 +1251,13 @@ "output_type": "stream", "text": [ "{\r\n", - " \"prepare_message_ts\": \"2021-04-26 10:39:59.268539\"\r\n", + " \"prepare_message_ts\": \"2023-04-20 12:10:28.197315\"\r\n", "}" ] } ], "source": [ - "cat my_runs/6/info.json" + "cat my_runs/5/info.json" ] }, { @@ -1326,7 +1290,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 38, "metadata": { "slideshow": { "slide_type": "slide" @@ -1338,7 +1302,7 @@ "output_type": "stream", "text": [ "INFO - resources - Running command 'my_main'\n", - "INFO - resources - Started run with ID \"3\"\n", + "INFO - resources - Started run with ID \"6\"\n", "INFO - resources - Completed after 0:00:00\n" ] }, @@ -1353,10 +1317,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 6, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -1379,7 +1343,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -1397,21 +1361,21 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 41, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "my_runs/3/run.json: \"name\": \"resources\",\r\n", - "my_runs/3/run.json: \"resources\": [\r\n", - "my_runs/3/run.json: \"my_runs/_resources/Iris_717820ef0af287ff346c5cabfb4c612c.csv\"\r\n" + "my_runs/6/run.json: \"name\": \"resources\",\r\n", + "my_runs/6/run.json: \"resources\": [\r\n", + "my_runs/6/run.json: \"my_runs/_resources/Iris_717820ef0af287ff346c5cabfb4c612c.csv\"\r\n" ] } ], "source": [ - "!grep -e \"resources\" -R my_runs/3" + "!grep -e \"resources\" -R my_runs/6" ] }, { @@ -1431,7 +1395,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 42, "metadata": { "slideshow": { "slide_type": "slide" @@ -1442,14 +1406,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Collecting pymongo\n", - " Downloading pymongo-4.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (492 kB)\n", - "\u001b[K |████████████████████████████████| 492 kB 1.2 MB/s eta 0:00:01\n", - "\u001b[?25hCollecting dnspython<3.0.0,>=1.16.0\n", - " Downloading dnspython-2.3.0-py3-none-any.whl (283 kB)\n", - "\u001b[K |████████████████████████████████| 283 kB 1.0 MB/s eta 0:00:01\n", - "\u001b[?25hInstalling collected packages: dnspython, pymongo\n", - "Successfully installed dnspython-2.3.0 pymongo-4.3.3\n" + "Requirement already satisfied: pymongo in /home/tomek/miniconda3/lib/python3.9/site-packages (4.3.3)\r\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from pymongo) (2.3.0)\r\n" ] } ], @@ -1470,7 +1428,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO - sacred_scopes - Running command 'my_main'\n" + "INFO - sacred_scopes - Running command 'my_main'\n", + "ERROR - sacred_scopes - Failed after 0:00:30!\n" ] } ], @@ -1479,7 +1438,7 @@ "from sacred import Experiment\n", "\n", "ex = Experiment(\"sacred_scopes\", interactive=True)\n", - "ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017',\n", + "ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017',\n", " db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.\n", "# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://admin:IUM_2021@172.17.0.1:27017\n", "@ex.config\n", @@ -1543,7 +1502,7 @@ "import time\n", "\n", "ex = Experiment(\"sacred_scopes\", interactive=True)\n", - "ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017',\n", + "ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017',\n", " db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.\n", "# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://admin:IUM_2021@172.17.0.1:27017\n", "@ex.config\n", diff --git a/IUM_08.MLFlow.ipynb b/IUM_08.MLFlow.ipynb index 86ad396..1c5e669 100644 --- a/IUM_08.MLFlow.ipynb +++ b/IUM_08.MLFlow.ipynb @@ -12,7 +12,7 @@ "
\n", "

Inżynieria uczenia maszynowego

\n", "

8. MLFlow [laboratoria]

\n", - "

Tomasz Ziętkiewicz (2022)

\n", + "

Tomasz Ziętkiewicz (2023)

\n", "
\n", "\n", "![Logo 2](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech2.jpg)"