AITech/aitech-ium
112
8
Fork 2
forked from tzietkiewicz/aitech-ium
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: AITech:8264284fd7c2217c41bbb59732ed4a559d6f24da
Branches Tags
AITech:master
tzietkiewicz:master
tzietkiewicz:sacred-mongo
tzietkiewicz:zaoczne
..
compare: AITech:1d3e3ff71c4348efc002363102a562094879a141
Branches Tags
AITech:master
tzietkiewicz:master
tzietkiewicz:sacred-mongo
tzietkiewicz:zaoczne

No commits in common. "8264284fd7c2217c41bbb59732ed4a559d6f24da" and "1d3e3ff71c4348efc002363102a562094879a141" have entirely different histories.
8264284fd7 ... 1d3e3ff71c

6 changed files with 248 additions and 182 deletions
Show Stats Expand all files Collapse all files

2
IUM_04.Konteneryzacja.ipynb
View File

@ -248,7 +248,7 @@
} }
}, },
"source": [ "source": [
"### Przykładowy Dockerfile:\n", "Przykładowy Dockerfile:\n",
"```Dockerfile\n", "```Dockerfile\n",
"# Nasz obraz będzie dzidziczył z obrazu Ubuntu w wersji latest\n", "# Nasz obraz będzie dzidziczył z obrazu Ubuntu w wersji latest\n",
"FROM ubuntu:latest\n", "FROM ubuntu:latest\n",

337
IUM_07.Sacred.ipynb
View File

@ -115,7 +115,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 2,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -126,18 +126,40 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Requirement already satisfied: sacred in /home/tomek/miniconda3/lib/python3.9/site-packages (0.8.4)\n", "Collecting sacred\n",
"Requirement already satisfied: jsonpickle>=1.2 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (3.0.1)\n", " Downloading sacred-0.8.4-py2.py3-none-any.whl (107 kB)\n",
"Requirement already satisfied: colorama>=0.4 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (0.4.6)\n", "\u001b[K |████████████████████████████████| 107 kB 637 kB/s eta 0:00:01\n",
"Requirement already satisfied: GitPython in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (3.1.31)\n", "\u001b[?25hCollecting py-cpuinfo>=4.0\n",
"Requirement already satisfied: py-cpuinfo>=4.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (9.0.0)\n", " Downloading py_cpuinfo-9.0.0-py3-none-any.whl (22 kB)\n",
"Requirement already satisfied: wrapt<2.0,>=1.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (1.15.0)\n", "Collecting GitPython\n",
"Requirement already satisfied: munch<3.0,>=2.5 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (2.5.0)\n", " Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\n",
"Requirement already satisfied: docopt<1.0,>=0.3 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (0.6.2)\n", "\u001b[K |████████████████████████████████| 184 kB 617 kB/s eta 0:00:01\n",
"Requirement already satisfied: packaging>=18.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (23.0)\n", "\u001b[?25hCollecting munch<3.0,>=2.5\n",
" Downloading munch-2.5.0-py2.py3-none-any.whl (10 kB)\n",
"Collecting wrapt<2.0,>=1.0\n",
" Downloading wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (78 kB)\n",
"\u001b[K |████████████████████████████████| 78 kB 508 kB/s eta 0:00:01\n",
"\u001b[?25hCollecting colorama>=0.4\n",
" Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
"Collecting docopt<1.0,>=0.3\n",
" Downloading docopt-0.6.2.tar.gz (25 kB)\n",
"Collecting jsonpickle>=1.2\n",
" Downloading jsonpickle-3.0.1-py2.py3-none-any.whl (40 kB)\n",
"\u001b[K |████████████████████████████████| 40 kB 518 kB/s eta 0:00:01\n",
"\u001b[?25hRequirement already satisfied: packaging>=18.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from sacred) (23.0)\n",
"Requirement already satisfied: six in /home/tomek/miniconda3/lib/python3.9/site-packages (from munch<3.0,>=2.5->sacred) (1.16.0)\n", "Requirement already satisfied: six in /home/tomek/miniconda3/lib/python3.9/site-packages (from munch<3.0,>=2.5->sacred) (1.16.0)\n",
"Requirement already satisfied: gitdb<5,>=4.0.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from GitPython->sacred) (4.0.10)\n", "Collecting gitdb<5,>=4.0.1\n",
"Requirement already satisfied: smmap<6,>=3.0.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from gitdb<5,>=4.0.1->GitPython->sacred) (5.0.0)\n" " Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n",
"\u001b[K |████████████████████████████████| 62 kB 430 kB/s eta 0:00:01\n",
"\u001b[?25hCollecting smmap<6,>=3.0.1\n",
" Using cached smmap-5.0.0-py3-none-any.whl (24 kB)\n",
"Building wheels for collected packages: docopt\n",
" Building wheel for docopt (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Created wheel for docopt: filename=docopt-0.6.2-py2.py3-none-any.whl size=13723 sha256=9ae6354916cc7817db186faf89036b67a6562e0391c04e901e72698d9ce5bd8b\n",
" Stored in directory: /home/tomek/.cache/pip/wheels/70/4a/46/1309fc853b8d395e60bafaf1b6df7845bdd82c95fd59dd8d2b\n",
"Successfully built docopt\n",
"Installing collected packages: smmap, gitdb, wrapt, py-cpuinfo, munch, jsonpickle, GitPython, docopt, colorama, sacred\n",
"Successfully installed GitPython-3.1.31 colorama-0.4.6 docopt-0.6.2 gitdb-4.0.10 jsonpickle-3.0.1 munch-2.5.0 py-cpuinfo-9.0.0 sacred-0.8.4 smmap-5.0.0 wrapt-1.15.0\n"
] ]
} }
], ],
@ -186,7 +208,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 14,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -240,7 +262,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": 15,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -278,7 +300,6 @@
" -f --force Disable warnings about suspicious changes for\r\n", " -f --force Disable warnings about suspicious changes for\r\n",
" this run.\r\n", " this run.\r\n",
" -h --help Print this help message and exit.\r\n", " -h --help Print this help message and exit.\r\n",
" -i VALUE --id=VALUE Set the id for this run.\r\n",
" -l VALUE --loglevel=VALUE Set the LogLevel. Loglevel either as 0 - 50 or\r\n", " -l VALUE --loglevel=VALUE Set the LogLevel. Loglevel either as 0 - 50 or\r\n",
" as string: DEBUG(10), INFO(20), WARNING(30),\r\n", " as string: DEBUG(10), INFO(20), WARNING(30),\r\n",
" ERROR(40), CRITICAL(50)\r\n", " ERROR(40), CRITICAL(50)\r\n",
@ -357,7 +378,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 5,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -386,7 +407,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": 6,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -413,10 +434,10 @@
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"<sacred.run.Run at 0x7f89cc3175b0>" "<sacred.run.Run at 0x7f641e9ddba8>"
] ]
}, },
"execution_count": 9, "execution_count": 6,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -438,7 +459,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 10, "execution_count": 19,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -451,7 +472,7 @@
"{'recipient': 'Świecie', 'greeting': 'Witaj', 'message': 'Witaj Świecie!'}" "{'recipient': 'Świecie', 'greeting': 'Witaj', 'message': 'Witaj Świecie!'}"
] ]
}, },
"execution_count": 10, "execution_count": 19,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -500,7 +521,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 21,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -525,7 +546,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 15, "execution_count": 24,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -542,7 +563,7 @@
" greeting = 'Witaj'\r\n", " greeting = 'Witaj'\r\n",
" message = 'Witaj Świecie!'\r\n", " message = 'Witaj Świecie!'\r\n",
" recipient = 'Świecie'\r\n", " recipient = 'Świecie'\r\n",
" seed = 269258424 \u001b[2m# the random seed for this experiment\u001b[0m\r\n", " seed = 539001265 \u001b[2m# the random seed for this experiment\u001b[0m\r\n",
"INFO - sacred_scopes - Completed after 0:00:00\r\n" "INFO - sacred_scopes - Completed after 0:00:00\r\n"
] ]
} }
@ -553,7 +574,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 16, "execution_count": 90,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -570,7 +591,7 @@
" greeting = 'Witaj'\r\n", " greeting = 'Witaj'\r\n",
" message = 'Witaj Przygodo!'\r\n", " message = 'Witaj Przygodo!'\r\n",
"\u001b[34m recipient = 'Przygodo'\u001b[0m\r\n", "\u001b[34m recipient = 'Przygodo'\u001b[0m\r\n",
" seed = 667939214 \u001b[2m# the random seed for this experiment\u001b[0m\r\n", " seed = 215765170 \u001b[2m# the random seed for this experiment\u001b[0m\r\n",
"INFO - sacred_scopes - Completed after 0:00:00\r\n" "INFO - sacred_scopes - Completed after 0:00:00\r\n"
] ]
} }
@ -592,13 +613,24 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 7,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
} }
}, },
"outputs": [], "outputs": [
{
"data": {
"text/plain": [
"{'recipient': 'samotności', 'greeting': 'Żegnaj'}"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"# %load IUM_07/config.json\n", "# %load IUM_07/config.json\n",
"{\n", "{\n",
@ -609,7 +641,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 20, "execution_count": 119,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -639,7 +671,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 21, "execution_count": 120,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -670,7 +702,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 22, "execution_count": 121,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -680,10 +712,10 @@
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"{'recipient': 'samotności', 'greeting': 'Żegnaj', 'seed': 877272352}" "{'recipient': 'samotności', 'greeting': 'Żegnaj', 'seed': 529757761}"
] ]
}, },
"execution_count": 22, "execution_count": 121,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -705,7 +737,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 23, "execution_count": 124,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -751,7 +783,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 24, "execution_count": 193,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -779,10 +811,10 @@
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"<sacred.run.Run at 0x7f89a0db5550>" "<sacred.run.Run at 0x7f423c40d820>"
] ]
}, },
"execution_count": 24, "execution_count": 193,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -832,24 +864,11 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 28, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"total 20\r\n",
"drwsrwsr-t 2 tomek tomek 4096 May 28 2022 1\r\n",
"drwsrwsr-t 2 tomek tomek 4096 May 28 2022 2\r\n",
"drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 3\r\n",
"drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 _resources\r\n",
"drwsrwsr-t 2 tomek tomek 4096 May 28 2022 _sources\r\n"
]
}
],
"source": [ "source": [
"!ls -l my_runs" "!ls -l runs"
] ]
}, },
{ {
@ -887,13 +906,21 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 9,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
} }
}, },
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting IUM_07/file_observer.py\n"
]
}
],
"source": [ "source": [
"%%writefile IUM_07/file_observer.py\n", "%%writefile IUM_07/file_observer.py\n",
"from sacred.observers import FileStorageObserver\n", "from sacred.observers import FileStorageObserver\n",
@ -919,7 +946,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 29, "execution_count": 11,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "fragment" "slide_type": "fragment"
@ -931,7 +958,7 @@
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"INFO - file_observer - Running command 'my_main'\r\n", "INFO - file_observer - Running command 'my_main'\r\n",
"INFO - file_observer - Started run with ID \"4\"\r\n", "INFO - file_observer - Started run with ID \"1\"\r\n",
"Witaj Świecie!\r\n", "Witaj Świecie!\r\n",
"INFO - file_observer - Completed after 0:00:00\r\n" "INFO - file_observer - Completed after 0:00:00\r\n"
] ]
@ -954,7 +981,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 30, "execution_count": 12,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -965,13 +992,9 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"total 24\r\n", "total 0\r\n",
"drwsrwsr-t 2 tomek tomek 4096 May 28 2022 1\r\n", "drwxr-xr-x 1 tomek tomek 512 Apr 25 09:51 1\r\n",
"drwsrwsr-t 2 tomek tomek 4096 May 28 2022 2\r\n", "drwxr-xr-x 1 tomek tomek 512 Apr 25 09:51 _sources\r\n"
"drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 3\r\n",
"drwxr-sr-x 2 tomek tomek 4096 Apr 20 12:09 4\r\n",
"drwxr-sr-x 2 tomek tomek 4096 Apr 12 15:11 _resources\r\n",
"drwsrwsr-t 2 tomek tomek 4096 May 28 2022 _sources\r\n"
] ]
} }
], ],
@ -981,13 +1004,25 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 14,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
} }
}, },
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"total 4\r\n",
"-rw-r--r-- 1 tomek tomek 77 Apr 25 09:51 config.json\r\n",
"-rw-r--r-- 1 tomek tomek 159 Apr 25 09:51 cout.txt\r\n",
"-rw-r--r-- 1 tomek tomek 2 Apr 25 09:51 metrics.json\r\n",
"-rw-r--r-- 1 tomek tomek 1659 Apr 25 09:51 run.json\r\n"
]
}
],
"source": [ "source": [
"!ls -l my_runs/1" "!ls -l my_runs/1"
] ]
@ -1012,7 +1047,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 31, "execution_count": 16,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -1118,7 +1153,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 32, "execution_count": 170,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -1130,7 +1165,7 @@
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"total 4\r\n", "total 4\r\n",
"-rw-r--r-- 1 tomek tomek 464 May 28 2022 file_observer_cd34a0ef4a32fb0a966eaa01ea6371ad.py\r\n" "-rw-rw-r-- 1 tomek tomek 463 kwi 26 10:21 file_observer_bb0a5c4720d1072b641d23da080696b6.py\r\n"
] ]
} }
], ],
@ -1149,7 +1184,8 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"# %load my_runs/_sources/file_observer_cd34a0ef4a32fb0a966eaa01ea6371ad.py\n", "## Źródła zostały zapisane\n",
"# %load my_runs/_sources/file_observer_bb0a5c4720d1072b641d23da080696b6.py\n",
"from sacred.observers import FileStorageObserver\n", "from sacred.observers import FileStorageObserver\n",
"from sacred import Experiment\n", "from sacred import Experiment\n",
"\n", "\n",
@ -1168,7 +1204,7 @@
"\n", "\n",
"@ex.automain\n", "@ex.automain\n",
"def my_main(recipient, greeting):\n", "def my_main(recipient, greeting):\n",
" print(prepare_message()) ## Nie musimy przekazywać wartości\n" " print(prepare_message()) ## Nie musimy przekazywać wartości"
] ]
}, },
{ {
@ -1184,7 +1220,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 36, "execution_count": 22,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -1196,7 +1232,7 @@
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"INFO - file_observer - Running command 'my_main'\n", "INFO - file_observer - Running command 'my_main'\n",
"INFO - file_observer - Started run with ID \"5\"\n", "INFO - file_observer - Started run with ID \"2\"\n",
"INFO - file_observer - Completed after 0:00:00\n" "INFO - file_observer - Completed after 0:00:00\n"
] ]
}, },
@ -1239,7 +1275,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 37, "execution_count": 185,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -1251,13 +1287,13 @@
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"{\r\n", "{\r\n",
" \"prepare_message_ts\": \"2023-04-20 12:10:28.197315\"\r\n", " \"prepare_message_ts\": \"2021-04-26 10:39:59.268539\"\r\n",
"}" "}"
] ]
} }
], ],
"source": [ "source": [
"cat my_runs/5/info.json" "cat my_runs/6/info.json"
] ]
}, },
{ {
@ -1290,7 +1326,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 38, "execution_count": 6,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -1302,7 +1338,7 @@
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"INFO - resources - Running command 'my_main'\n", "INFO - resources - Running command 'my_main'\n",
"INFO - resources - Started run with ID \"6\"\n", "INFO - resources - Started run with ID \"3\"\n",
"INFO - resources - Completed after 0:00:00\n" "INFO - resources - Completed after 0:00:00\n"
] ]
}, },
@ -1317,10 +1353,10 @@
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"<sacred.run.Run at 0x7f89a0db5a30>" "<sacred.run.Run at 0x7f9b8821cd90>"
] ]
}, },
"execution_count": 38, "execution_count": 6,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -1343,7 +1379,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 39, "execution_count": 7,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1361,21 +1397,21 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 41, "execution_count": 9,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"my_runs/6/run.json: \"name\": \"resources\",\r\n", "my_runs/3/run.json: \"name\": \"resources\",\r\n",
"my_runs/6/run.json: \"resources\": [\r\n", "my_runs/3/run.json: \"resources\": [\r\n",
"my_runs/6/run.json: \"my_runs/_resources/Iris_717820ef0af287ff346c5cabfb4c612c.csv\"\r\n" "my_runs/3/run.json: \"my_runs/_resources/Iris_717820ef0af287ff346c5cabfb4c612c.csv\"\r\n"
] ]
} }
], ],
"source": [ "source": [
"!grep -e \"resources\" -R my_runs/6" "!grep -e \"resources\" -R my_runs/3"
] ]
}, },
{ {
@ -1395,7 +1431,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 42, "execution_count": 19,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -1406,8 +1442,11 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Requirement already satisfied: pymongo in /home/tomek/miniconda3/lib/python3.9/site-packages (4.3.3)\r\n", "Collecting pymongo\n",
"Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from pymongo) (2.3.0)\r\n" " Downloading https://files.pythonhosted.org/packages/10/3b/46541b4ee3000019b8ef5b1847292ddc77f492c162bc4d49c424db7fc97a/pymongo-4.1.1-cp36-cp36m-manylinux1_x86_64.whl (464kB)\n",
"\u001b[K 100% |████████████████████████████████| 471kB 959kB/s ta 0:00:01\n",
"\u001b[?25hInstalling collected packages: pymongo\n",
"Successfully installed pymongo-4.1.1\n"
] ]
} }
], ],
@ -1417,7 +1456,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 23,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -1429,7 +1468,76 @@
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"INFO - sacred_scopes - Running command 'my_main'\n", "INFO - sacred_scopes - Running command 'my_main'\n",
"ERROR - sacred_scopes - Failed after 0:00:30!\n" "ERROR - sacred_scopes - Failed after 0:00:30!\n",
"ERROR - sacred_scopes - Traceback (most recent call last):\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/run.py\", line 235, in __call__\n",
" self._emit_started()\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/run.py\", line 333, in _emit_started\n",
" _id=self._id,\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\", line 258, in started_event\n",
" self.insert()\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\", line 367, in insert\n",
" c.next()[\"_id\"] + 1 if self.runs.count_documents({}, limit=1) else 1\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/collection.py\", line 1811, in count_documents\n",
" return self._retryable_non_cursor_read(_cmd, session)\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/collection.py\", line 1816, in _retryable_non_cursor_read\n",
" with client._tmp_session(session) as s:\n",
" File \"/usr/lib/python3.6/contextlib.py\", line 81, in __enter__\n",
" return next(self.gen)\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\", line 1676, in _tmp_session\n",
" s = self._ensure_session(session)\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\", line 1663, in _ensure_session\n",
" return self.__start_session(True, causal_consistency=False)\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\", line 1608, in __start_session\n",
" self._topology._check_implicit_session_support()\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/topology.py\", line 519, in _check_implicit_session_support\n",
" self._check_session_support()\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/topology.py\", line 536, in _check_session_support\n",
" readable_server_selector, self._settings.server_selection_timeout, None\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/pymongo/topology.py\", line 229, in _select_servers_loop\n",
" % (self._error_message(selector), timeout, self.description)\n",
"pymongo.errors.ServerSelectionTimeoutError: localhost:27017: [Errno 111] Connection refused, Timeout: 30s, Topology Description: <TopologyDescription id: 6266603d3269f60a67f61383, topology_type: Unknown, servers: [<ServerDescription ('localhost', 27017) server_type: Unknown, rtt: None, error=AutoReconnect('localhost:27017: [Errno 111] Connection refused',)>]>\n",
"\n",
"During handling of the above exception, another exception occurred:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/run.py\", line 429, in _final_call\n",
" getattr(observer, method)(**kwargs)\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\", line 283, in failed_event\n",
" self.final_save(attempts=1)\n",
" File \"/home/tomek/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\", line 421, in final_save\n",
" os.makedirs(self.failure_dir, exist_ok=True)\n",
" File \"/usr/lib/python3.6/os.py\", line 205, in makedirs\n",
" head, tail = path.split(name)\n",
" File \"/usr/lib/python3.6/posixpath.py\", line 107, in split\n",
" p = os.fspath(p)\n",
"TypeError: expected str, bytes or os.PathLike object, not NoneType\n",
"\n"
]
},
{
"ename": "ServerSelectionTimeoutError",
"evalue": "localhost:27017: [Errno 111] Connection refused, Timeout: 30s, Topology Description: <TopologyDescription id: 6266603d3269f60a67f61383, topology_type: Unknown, servers: [<ServerDescription ('localhost', 27017) server_type: Unknown, rtt: None, error=AutoReconnect('localhost:27017: [Errno 111] Connection refused',)>]>",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mServerSelectionTimeoutError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-23-95f1b404dde7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprepare_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m## Nie musimy przekazywać wartości\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/sacred/experiment.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, command_name, config_updates, named_configs, info, meta_info, options)\u001b[0m\n\u001b[1;32m 274\u001b[0m \u001b[0mcommand_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig_updates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnamed_configs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta_info\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 275\u001b[0m )\n\u001b[0;32m--> 276\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 277\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/sacred/run.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 233\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 234\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mcapture_stdout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_emit_started\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 236\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_heartbeat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 237\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_execute_pre_run_hooks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/sacred/run.py\u001b[0m in \u001b[0;36m_emit_started\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 331\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 332\u001b[0m \u001b[0mmeta_info\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmeta_info\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m \u001b[0m_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 334\u001b[0m )\n\u001b[1;32m 335\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_id\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\u001b[0m in \u001b[0;36mstarted_event\u001b[0;34m(self, ex_info, command, host_info, start_time, config, meta_info, _id)\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;31m# save sources\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_entry\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"experiment\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"sources\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_sources\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mex_info\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 258\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 259\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_entry\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"_id\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/sacred/observers/mongo.py\u001b[0m in \u001b[0;36minsert\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 365\u001b[0m \u001b[0mc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"_id\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpymongo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDESCENDING\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlimit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 366\u001b[0m self.run_entry[\"_id\"] = (\n\u001b[0;32m--> 367\u001b[0;31m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"_id\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mruns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcount_documents\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 368\u001b[0m )\n\u001b[1;32m 369\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/collection.py\u001b[0m in \u001b[0;36mcount_documents\u001b[0;34m(self, filter, session, comment, **kwargs)\u001b[0m\n\u001b[1;32m 1809\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"n\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1811\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_retryable_non_cursor_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_cmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1812\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1813\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_retryable_non_cursor_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/collection.py\u001b[0m in \u001b[0;36m_retryable_non_cursor_read\u001b[0;34m(self, func, session)\u001b[0m\n\u001b[1;32m 1814\u001b[0m \u001b[0;34m\"\"\"Non-cursor read helper to handle implicit session creation.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1815\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__database\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1816\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tmp_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1817\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_retryable_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_preference_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1818\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/lib/python3.6/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 79\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 82\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 83\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m_tmp_session\u001b[0;34m(self, session, close)\u001b[0m\n\u001b[1;32m 1674\u001b[0m \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1676\u001b[0;31m \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ensure_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1677\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1678\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m_ensure_session\u001b[0;34m(self, session)\u001b[0m\n\u001b[1;32m 1661\u001b[0m \u001b[0;31m# Don't make implicit sessions causally consistent. Applications\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1662\u001b[0m \u001b[0;31m# should always opt-in.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1663\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__start_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcausal_consistency\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1664\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mConfigurationError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mInvalidOperation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1665\u001b[0m \u001b[0;31m# Sessions not supported.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m__start_session\u001b[0;34m(self, implicit, **kwargs)\u001b[0m\n\u001b[1;32m 1606\u001b[0m \u001b[0;31m# Raises ConfigurationError if sessions are not supported.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1607\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mimplicit\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1608\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_topology\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_implicit_session_support\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1609\u001b[0m \u001b[0mserver_session\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_EmptyServerSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1610\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/topology.py\u001b[0m in \u001b[0;36m_check_implicit_session_support\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 517\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_check_implicit_session_support\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 518\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 519\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_session_support\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 520\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 521\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_check_session_support\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/topology.py\u001b[0m in \u001b[0;36m_check_session_support\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 534\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_description\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable_servers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 535\u001b[0m self._select_servers_loop(\n\u001b[0;32m--> 536\u001b[0;31m \u001b[0mreadable_server_selector\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_settings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserver_selection_timeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 537\u001b[0m )\n\u001b[1;32m 538\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.local/lib/python3.6/site-packages/pymongo/topology.py\u001b[0m in \u001b[0;36m_select_servers_loop\u001b[0;34m(self, selector, timeout, address)\u001b[0m\n\u001b[1;32m 227\u001b[0m raise ServerSelectionTimeoutError(\n\u001b[1;32m 228\u001b[0m \u001b[0;34m\"%s, Timeout: %ss, Topology Description: %r\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 229\u001b[0;31m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_error_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselector\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdescription\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 230\u001b[0m )\n\u001b[1;32m 231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mServerSelectionTimeoutError\u001b[0m: localhost:27017: [Errno 111] Connection refused, Timeout: 30s, Topology Description: <TopologyDescription id: 6266603d3269f60a67f61383, topology_type: Unknown, servers: [<ServerDescription ('localhost', 27017) server_type: Unknown, rtt: None, error=AutoReconnect('localhost:27017: [Errno 111] Connection refused',)>]>"
] ]
} }
], ],
@ -1438,9 +1546,9 @@
"from sacred import Experiment\n", "from sacred import Experiment\n",
"\n", "\n",
"ex = Experiment(\"sacred_scopes\", interactive=True)\n", "ex = Experiment(\"sacred_scopes\", interactive=True)\n",
"ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017',\n", "ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017',\n",
" db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.\n", " db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.\n",
"# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://admin:IUM_2021@172.17.0.1:27017\n", "# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://mongo_user:mongo_password_IUM_2021@localhost:27017\n",
"@ex.config\n", "@ex.config\n",
"def my_config():\n", "def my_config():\n",
" recipient = \"Świecie\"\n", " recipient = \"Świecie\"\n",
@ -1488,13 +1596,46 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 192,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
} }
}, },
"outputs": [], "outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO - sacred_scopes - Running command 'my_main'\n",
"INFO - sacred_scopes - Started run with ID \"9\"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Witaj Świecie!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO - sacred_scopes - Completed after 0:00:50\n"
]
},
{
"data": {
"text/plain": [
"<sacred.run.Run at 0x7f423c2de550>"
]
},
"execution_count": 192,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"from sacred.observers import MongoObserver\n", "from sacred.observers import MongoObserver\n",
"from sacred import Experiment\n", "from sacred import Experiment\n",
@ -1502,9 +1643,9 @@
"import time\n", "import time\n",
"\n", "\n",
"ex = Experiment(\"sacred_scopes\", interactive=True)\n", "ex = Experiment(\"sacred_scopes\", interactive=True)\n",
"ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017',\n", "ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017',\n",
" db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.\n", " db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.\n",
"# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://admin:IUM_2021@172.17.0.1:27017\n", "# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://mongo_user:mongo_password_IUM_2021@localhost:27017\n",
"@ex.config\n", "@ex.config\n",
"def my_config():\n", "def my_config():\n",
" recipient = \"Świecie\"\n", " recipient = \"Świecie\"\n",

9
IUM_07/Dockerfile
View File

@ -1,9 +0,0 @@
FROM ubuntu:latest
RUN apt update && apt install -y \
git \
python3-pip \
python3
RUN python3 -m pip install sacred pymongo

13
IUM_07/Jenkinsfile vendored
View File

@ -1,13 +0,0 @@
node {
checkout scm
//Pierwszy argument to tag, który zostania nadany zbudowanemu obrazowi
//Jeśli chcemy użyć Dockerfile z innej ścieżki niż ./Dockerfile, możemy ją podać jako drugi argument
def testImage = docker.build("sacred_pymongo", "./IUM_07/")
//Wszystkie polecenia poniżej wykonają się w kontenerze, z podmontowanym Workspace Jenkinsa
testImage.inside {
dir ("IUM_07"){
sh 'python3 mongo_observer.py'
}
}
}

39
IUM_07/mongo_observer.py
View File

@ -1,39 +0,0 @@
from sacred.observers import MongoObserver
from sacred import Experiment
import random
import time
ex = Experiment("sacred_scopes", interactive=True)
ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017', db_name='sacred'))
#ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password@localhost:27017',
# db_name='sacred')) # Tutaj podajemy dane uwierzytelniające i nazwę bazy skonfigurowane w pliku .env podczas uruchamiania bazy.
# W przypadku instancji na Jenkinsie url będzie wyglądał następująco: mongodb://mongo_user:mongo_password_IUM_2021@localhost:27017
@ex.config
def my_config():
recipient = "Świecie"
greeting = "Witaj"
@ex.capture
def prepare_message(recipient, greeting):
return "{0} {1}!".format(greeting, recipient)
@ex.main
def my_main(recipient, greeting, _run):
print(prepare_message()) ## Nie musimy przekazywać wartości
counter = 0
while counter < 20:
counter+=1
value = counter
ms_to_wait = random.randint(5, 5000)
time.sleep(ms_to_wait/1000)
noise = 1.0 + 0.1 * (random.randint(0, 10) - 5)
# This will add an entry for training.loss metric in every second iteration.
# The resulting sequence of steps for training.loss will be 0, 2, 4, ...
if counter % 2 == 0:
_run.log_scalar("training.loss", value * 1.5 * noise, counter)
# Implicit step counter (0, 1, 2, 3, ...)
# incremented with each call for training.accuracy:
_run.log_scalar("training.accuracy", value * 2 * noise)
ex.run()

30
IUM_08.MLFlow.ipynb
View File

@ -8,11 +8,11 @@
} }
}, },
"source": [ "source": [
"## ![Logo 1](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech1.jpg)\n", "![Logo 1](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech1.jpg)\n",
"<div class=\"alert alert-block alert-info\">\n", "<div class=\"alert alert-block alert-info\">\n",
"<h1> Inżynieria uczenia maszynowego </h1>\n", "<h1> Inżynieria uczenia maszynowego </h1>\n",
"<h2> 8. <i>MLFlow</i> [laboratoria]</h2> \n", "<h2> 8. <i>MLFlow</i> [laboratoria]</h2> \n",
"<h3> Tomasz Ziętkiewicz (2023)</h3>\n", "<h3> Tomasz Ziętkiewicz (2022)</h3>\n",
"</div>\n", "</div>\n",
"\n", "\n",
"![Logo 2](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech2.jpg)" "![Logo 2](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech2.jpg)"
@ -73,7 +73,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 1,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -88,16 +88,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 80,
"metadata": {},
"outputs": [],
"source": [
"!mkdir -p IUM_08/examples/sklearn_elasticnet_wine/"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"
@ -199,7 +190,6 @@
" mlflow.log_metric(\"mae\", mae)\n", " mlflow.log_metric(\"mae\", mae)\n",
" \n", " \n",
" # Infer model signature to log it\n", " # Infer model signature to log it\n",
" # Więcej o sygnaturach: https://mlflow.org/docs/latest/models.html?highlight=signature#model-signature\n",
" signature = mlflow.models.signature.infer_signature(train_x, lr.predict(train_x))\n", " signature = mlflow.models.signature.infer_signature(train_x, lr.predict(train_x))\n",
"\n", "\n",
" tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme\n", " tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme\n",
@ -958,7 +948,7 @@
"source": [ "source": [
"# Zadania [10p pkt]\n", "# Zadania [10p pkt]\n",
"1. Dodaj do swojego projektu logowanie parametrów i metryk za pomocą MLflow (polecenia `mlflow.log_param` i `mlflow.log_metric`\n", "1. Dodaj do swojego projektu logowanie parametrów i metryk za pomocą MLflow (polecenia `mlflow.log_param` i `mlflow.log_metric`\n",
"2. Dodaj plik MLProject definiujący polecenia do trenowania i testowania, ich parametry wywołania oraz środowisko (Conda albo Docker)" "2. Dodaj plik MLProject definiujący polecenia do trenowania i testowania, ich parametry wywołania oraz środowisko (użyj zdefiniowanego wcześniej obrazu Docker)"
] ]
}, },
{ {
@ -1400,7 +1390,7 @@
} }
}, },
"source": [ "source": [
"## Zadania (termin: 14.05 EOD) \n", "## Zadania\n",
"1. [2 pkt] Dodaj do joba treningowego wywołania MLflow, tak, żeby przy każdym uruchomieniu stworzyć i zarchiwizować katalog z modelem. Plik MLmodel powinien zawierać pola:\n", "1. [2 pkt] Dodaj do joba treningowego wywołania MLflow, tak, żeby przy każdym uruchomieniu stworzyć i zarchiwizować katalog z modelem. Plik MLmodel powinien zawierać pola:\n",
" - signature\n", " - signature\n",
" - input_example\n", " - input_example\n",
@ -1435,10 +1425,6 @@
" - CLI: `export MLFLOW_TRACKING_URI=http://172.17.0.1:5000`\n", " - CLI: `export MLFLOW_TRACKING_URI=http://172.17.0.1:5000`\n",
" \n", " \n",
"- Żeby klient MLflow działający w kontenerze docker mógł zapisywać i pdczytywać artefakty, muszą Państwo podmonotwać katalog `/tmp/mlruns` i `/mlruns` (ten drugi po to, żeby po restarcie serwera katalog nie został wyczyszczony)\n", "- Żeby klient MLflow działający w kontenerze docker mógł zapisywać i pdczytywać artefakty, muszą Państwo podmonotwać katalog `/tmp/mlruns` i `/mlruns` (ten drugi po to, żeby po restarcie serwera katalog nie został wyczyszczony)\n",
" - jak podmontować: ```docker.image('my-image').inside('-v /tmp/mlruns:/tmp/mlruns' -v /mlruns:/mlruns')```\n",
" - Przykład działąjącego joba na Jenkinsie zapisującego wyniki do MLFlow: https://tzietkiewicz.vm.wmi.amu.edu.pl:8081/job/gitea-test/job/ium-helloworld/job/master/\n",
" - Repo: https://git.wmi.amu.edu.pl/tzietkiewicz/ium-helloworld/src/branch/master\n",
"\n",
"- Proszę ustawić nazwę eksperymentu na numer indeksu, dzięki temu każdy z Państwa będzie widział swoje eksperymenty oddzielnie:\n", "- Proszę ustawić nazwę eksperymentu na numer indeksu, dzięki temu każdy z Państwa będzie widział swoje eksperymenty oddzielnie:\n",
"`mlflow.set_experiment(\"s123456\")`" "`mlflow.set_experiment(\"s123456\")`"
] ]
@ -1449,7 +1435,7 @@
"celltoolbar": "Slideshow", "celltoolbar": "Slideshow",
"email": "tomasz.zietkiewicz@amu.edu.pl", "email": "tomasz.zietkiewicz@amu.edu.pl",
"kernelspec": { "kernelspec": {
"display_name": "Python 3", "display_name": "Python 3 (ipykernel)",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@ -1464,7 +1450,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.9.12" "version": "3.10.4"
}, },
"slideshow": { "slideshow": {
"slide_type": "slide" "slide_type": "slide"