diff --git a/IUM_10.DVC.ipynb b/IUM_10.DVC.ipynb index fba8869..5d383f3 100644 --- a/IUM_10.DVC.ipynb +++ b/IUM_10.DVC.ipynb @@ -13,7 +13,7 @@ "
\n", "

Inżynieria uczenia maszynowego

\n", "

10. DVC [laboratoria]

\n", - "

Tomasz Ziętkiewicz (2022)

\n", + "

Tomasz Ziętkiewicz (2023)

\n", "
\n", "\n", "![Logo 2](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech2.jpg)" @@ -53,6 +53,41 @@ "- 5-minutowe wprowadzenie: https://www.youtube.com/watch?v=UbL7VUpv1Bs&t=197s" ] }, + { + "cell_type": "markdown", + "id": "3d4ce1cb", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Śledzenie plików za pomocą DVC\n", + " - dużymi plikami, takimi jak plikami z danymi wejściowymi czy plikami modeli, trudno zarządza się za pomocą gita, ze względu na problemy z:\n", + " - wydajnością\n", + " - przestrzenią w repozytorium\n", + " - ograniczenia ze strony serwisu (np. [limit 100 MB na plik w Github](https://docs.github.com/en/repositories/working-with-files/managing-large-files/about-large-files-on-github))\n", + " - Git posiada rozszerzenie [lfs(Large File Storage)](https://git-lfs.github.com/), które stanowi pewne rozwiązanie tego problemu. \n", + " - Same pliki przechowywane są na specjalnym zdalnym serwerze, w repozytorium przechowywane są jedynie odnośniki do tych plików i pewne metadane\n", + " - Github ma zintegrowany LFS z [limitem 1GB dla kont bezpłatnych](https://docs.github.com/en/repositories/working-with-files/managing-large-files/about-storage-and-bandwidth-usage)" + ] + }, + { + "cell_type": "markdown", + "id": "dd8e529b", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + " - DVC proponuje podobne podejście co LFS, ale:\n", + " - pliki mogą być przechowywane na niemal dowolnym serwerze, również lokalnie\n", + " - brak limitu wielkości plików (w Git-LFS na Github [limit 2GB](https://docs.github.com/en/repositories/working-with-files/managing-large-files/about-git-large-file-storage))\n", + " - DVC zapewnia dodatkowe narzędzie umożliwiające śledzenie plików i ich powiązań z wynikami eksperymentów\n", + " - więcej, patrz [tutaj](https://dvc.org/doc/user-guide/related-technologies)" + ] + }, { "cell_type": "markdown", "id": "9bfb356e", @@ -70,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "054c7a11", "metadata": { "slideshow": { @@ -83,421 +118,180 @@ "output_type": "stream", "text": [ "Collecting dvc\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/96/9d/9cca62742cb99da8002c9d6ac6c7463344deb60a219ca4f3a9778b02c67b/dvc-2.8.1-py3-none-any.whl (386kB)\n", - "\u001b[K 100% |████████████████████████████████| 389kB 1.4MB/s ta 0:00:01\n", - "\u001b[?25hCollecting setuptools>=34.0.0 (from dvc)\n", - " Using cached https://files.pythonhosted.org/packages/b0/3a/88b210db68e56854d0bcf4b38e165e03be377e13907746f825790f3df5bf/setuptools-59.6.0-py3-none-any.whl\n", - "Collecting colorama>=0.3.9 (from dvc)\n", - " Using cached https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl\n", - "Collecting dictdiffer>=0.8.1 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/47/ef/4cb333825d10317a36a1154341ba37e6e9c087bac99c1990ef07ffdb376f/dictdiffer-0.9.0-py2.py3-none-any.whl\n", - "Collecting ply>=3.9 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl (49kB)\n", - "\u001b[K 100% |████████████████████████████████| 51kB 1.8MB/s ta 0:00:01\n", - "\u001b[?25hCollecting nanotime>=0.5.2 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/d5/54/6d5924f59cf671326e7809f4b3f70fa8df535d67e952ad0b6fea02f52faf/nanotime-0.5.2.tar.gz\n", - "Collecting fsspec[http]>=2021.10.0 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/f6/90/32e53b96067954c2f916667f5a11634aeabef8ed70e83133ed8037b8111b/fsspec-2022.1.0-py3-none-any.whl (133kB)\n", - "\u001b[K 100% |████████████████████████████████| 143kB 1.3MB/s ta 0:00:01\n", - "\u001b[?25hCollecting networkx>=2.5 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/f3/b7/c7f488101c0bb5e4178f3cde416004280fd40262433496830de8a8c21613/networkx-2.5.1-py3-none-any.whl (1.6MB)\n", - "\u001b[K 100% |████████████████████████████████| 1.6MB 609kB/s ta 0:00:01\n", - "\u001b[?25hCollecting flufl.lock<4,>=3.2 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/1e/68/393c148df629f90a919de653ebb967a8bd8c83d07d2bc3150ca0faff3940/flufl.lock-3.2.tar.gz\n", - "Collecting shtab<2,>=1.3.4 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/37/c0/7d887589bb87db6b87b55315ac41acbbe7355074006a0968343d52ded5cd/shtab-1.5.4-py2.py3-none-any.whl\n", - "Collecting shortuuid>=0.5.0 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/01/14/df1aa61e1bb75a6fff19b25d7d175b7f188dc15a70dfc80e7badd5bda1de/shortuuid-1.0.9-py3-none-any.whl\n", - "Collecting gitpython>3 (from dvc)\n", - " Using cached https://files.pythonhosted.org/packages/55/60/f884f01eef2a7255875862ec1b12d57d74113ec6e8d9e16c4d254cd6aa3c/GitPython-3.1.20-py3-none-any.whl\n", - "Collecting tqdm<5,>=4.45.0 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/8a/c4/d15f1e627fff25443ded77ea70a7b5532d6371498f9285d44d62587e209c/tqdm-4.64.0-py2.py3-none-any.whl (78kB)\n", - "\u001b[K 100% |████████████████████████████████| 81kB 2.0MB/s ta 0:00:01\n", - "\u001b[?25hCollecting zc.lockfile>=1.2.1 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/6c/2a/268389776288f0f26c7272c70c36c96dcc0bdb88ab6216ea18e19df1fadd/zc.lockfile-2.0-py2.py3-none-any.whl\n", - "Collecting rich>=10.9.0 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/05/c0/844c879d659e15c4b1fbe8be89b25de7cf9b0d5e8485aadfe83ae6e7286a/rich-12.4.1-py3-none-any.whl (231kB)\n", - "\u001b[K 100% |████████████████████████████████| 235kB 1.5MB/s ta 0:00:01\n", - "\u001b[?25hCollecting aiohttp-retry>=2.4.5 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/0f/fe/58130a432d4397e174b82e57d8a11ccf5066631d03949dada4aac8b88041/aiohttp_retry-2.4.6-py3-none-any.whl\n", - "Collecting voluptuous>=0.11.7 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/a7/68/927add5dfd55a0d666ffc8939ff4390b76ca3ffbc36c12369f9a034393cb/voluptuous-0.13.1-py3-none-any.whl\n", - "Collecting psutil>=5.8.0 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/d6/de/0999ea2562b96d7165812606b18f7169307b60cd378bc29cf3673322c7e9/psutil-5.9.1.tar.gz (479kB)\n", - "\u001b[K 100% |████████████████████████████████| 481kB 1.2MB/s ta 0:00:011\n", - "\u001b[?25hCollecting flatten-dict<1,>=0.4.1 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/43/f5/ee39c6e92acc742c052f137b47c210cd0a1b72dcd3f98495528bb4d27761/flatten_dict-0.4.2-py2.py3-none-any.whl\n", - "Collecting configobj>=5.0.6 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/64/61/079eb60459c44929e684fa7d9e2fdca403f67d64dd9dbac27296be2e0fab/configobj-5.0.6.tar.gz\n", - "Collecting pygtrie>=2.3.2 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/a5/8b/90d0f21a27a354e808a73eb0ffb94db990ab11ad1d8b3db3e5196c882cad/pygtrie-2.4.2.tar.gz\n", - "Collecting tabulate>=0.8.7 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/ca/80/7c0cad11bd99985cfe7c09427ee0b4f9bd6b048bd13d4ffb32c6db237dfb/tabulate-0.8.9-py3-none-any.whl\n", - "Collecting dulwich>=0.20.23 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/50/88/d5f6b478086a78995a71151a956ac97d6635f0f6d80f625c96ba13cc612d/dulwich-0.20.40.tar.gz (423kB)\n", - "\u001b[K 100% |████████████████████████████████| 430kB 1.1MB/s ta 0:00:01\n", - "\u001b[?25hCollecting appdirs>=1.4.3 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl\n", - "Collecting ruamel.yaml>=0.17.11 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/9e/cb/938214ac358fbef7058343b3765c79a1b7ed0c366f7f992ce7ff38335652/ruamel.yaml-0.17.21-py3-none-any.whl (109kB)\n", - "\u001b[K 100% |████████████████████████████████| 112kB 2.1MB/s ta 0:00:01\n", - "\u001b[?25hCollecting python-benedict>=0.24.2 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/77/84/6268f9c31140f23ef290c89cb91197f5ea6cbbf102a15db13c1eea845a22/python_benedict-0.25.1-py3-none-any.whl (41kB)\n", - "\u001b[K 100% |████████████████████████████████| 51kB 2.2MB/s ta 0:00:01\n", - "\u001b[?25hCollecting grandalf==0.6 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/54/f4/a0b6a4c6d616d0a838b2dd0bc7bf74d73e8e8cdc880bab7fdb5fdc3d0e06/grandalf-0.6-py3-none-any.whl\n", - "Collecting pyparsing==2.4.7 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/8a/bb/488841f56197b13700afd5658fc279a2025a39e22449b7cf29864669b15d/pyparsing-2.4.7-py2.py3-none-any.whl (67kB)\n", - "\u001b[K 100% |████████████████████████████████| 71kB 2.1MB/s ta 0:00:01\n", - "\u001b[?25hCollecting dpath<3,>=2.0.2 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/9d/68/19b2a579121446ba80aa3574f04de6931ad549a22cd5588e43c8031de7f8/dpath-2.0.6-py3-none-any.whl\n", - "Collecting pygit2<1.7,>=1.5.0; python_version < \"3.7\" (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/6b/23/a8c5b726a58282fe2cadcc63faaddd4be147c3c8e0bd38b233114adf98fd/pygit2-1.6.1.tar.gz (258kB)\n", - "\u001b[K 100% |████████████████████████████████| 266kB 3.5MB/s eta 0:00:01\n", - "\u001b[?25hCollecting dataclasses>=0.7; python_version < \"3.7\" (from dvc)\n", - " Using cached https://files.pythonhosted.org/packages/fe/ca/75fac5856ab5cfa51bbbcefa250182e50441074fdc3f803f6e76451fab43/dataclasses-0.8-py3-none-any.whl\n", - "Collecting pyasn1>=0.4.1 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n" + " Downloading dvc-2.55.0-py3-none-any.whl (419 kB)\n", + "\u001b[K |████████████████████████████████| 419 kB 794 kB/s eta 0:00:01\n", + "\u001b[?25hCollecting funcy>=1.14\n", + " Downloading funcy-2.0-py2.py3-none-any.whl (30 kB)\n", + "Collecting voluptuous>=0.11.7\n", + " Using cached voluptuous-0.13.1-py3-none-any.whl (29 kB)\n", + "Collecting dvc-http>=2.29.0\n", + " Downloading dvc_http-2.30.2-py3-none-any.whl (12 kB)\n", + "Requirement already satisfied: colorama>=0.3.9 in /home/tomek/miniconda3/lib/python3.9/site-packages (from dvc) (0.4.6)\n", + "Collecting pathspec>=0.10.3\n", + " Downloading pathspec-0.11.1-py3-none-any.whl (29 kB)\n", + "Collecting pygtrie>=2.3.2\n", + " Downloading pygtrie-2.5.0-py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: ruamel.yaml>=0.17.11 in /home/tomek/miniconda3/lib/python3.9/site-packages (from dvc) (0.17.21)\n", + "Requirement already satisfied: tabulate>=0.8.7 in /home/tomek/miniconda3/lib/python3.9/site-packages (from dvc) (0.9.0)\n", + "Collecting zc.lockfile>=1.2.1\n", + " Downloading zc.lockfile-3.0.post1-py3-none-any.whl (9.8 kB)\n", + "Collecting dpath<3,>=2.1.0\n", + " Downloading dpath-2.1.5-py3-none-any.whl (17 kB)\n", + "Collecting shtab<2,>=1.3.4\n", + " Downloading shtab-1.6.1-py3-none-any.whl (13 kB)\n", + "Requirement already satisfied: tqdm<5,>=4.63.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from dvc) (4.64.0)\n", + "Collecting pydot>=1.2.4\n", + " Using cached pydot-1.4.2-py2.py3-none-any.whl (21 kB)\n", + "Collecting scmrepo<2,>=1.0.0\n", + " Downloading scmrepo-1.0.2-py3-none-any.whl (54 kB)\n", + "\u001b[K |████████████████████████████████| 54 kB 1.8 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting flatten-dict<1,>=0.4.1\n", + " Using cached flatten_dict-0.4.2-py2.py3-none-any.whl (9.7 kB)\n", + "Collecting psutil>=5.8\n", + " Downloading psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (282 kB)\n", + "\u001b[K |████████████████████████████████| 282 kB 21.9 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting dvc-data<0.48,>=0.47.1\n", + " Downloading dvc_data-0.47.2-py3-none-any.whl (59 kB)\n", + "\u001b[K |████████████████████████████████| 59 kB 4.1 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting dvc-render<0.4.0,>=0.3.1\n", + " Downloading dvc_render-0.3.1-py3-none-any.whl (18 kB)\n", + "Collecting dvc-studio-client<1,>=0.6.1\n", + " Downloading dvc_studio_client-0.8.0-py3-none-any.whl (10 kB)\n", + "Collecting flufl.lock>=5\n", + " Downloading flufl.lock-7.1.1-py3-none-any.whl (11 kB)\n", + "Requirement already satisfied: platformdirs<4,>=3.1.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from dvc) (3.1.1)\n", + "Collecting networkx>=2.5\n", + " Downloading networkx-3.1-py3-none-any.whl (2.1 MB)\n", + "\u001b[K |████████████████████████████████| 2.1 MB 14.1 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting grandalf<1,>=0.7\n", + " Downloading grandalf-0.8-py3-none-any.whl (41 kB)\n", + "\u001b[K |████████████████████████████████| 41 kB 304 kB/s eta 0:00:01\n", + "\u001b[?25hCollecting hydra-core>=1.1\n", + " Downloading hydra_core-1.3.2-py3-none-any.whl (154 kB)\n", + "\u001b[K |████████████████████████████████| 154 kB 14.3 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: pyparsing>=2.4.7 in /home/tomek/.local/lib/python3.9/site-packages (from dvc) (3.0.9)\n", + "Collecting tomlkit>=0.11.1\n", + " Downloading tomlkit-0.11.7-py3-none-any.whl (35 kB)\n", + "Requirement already satisfied: requests>=2.22 in /home/tomek/miniconda3/lib/python3.9/site-packages (from dvc) (2.27.1)\n", + "Requirement already satisfied: packaging>=19 in /home/tomek/miniconda3/lib/python3.9/site-packages (from dvc) (23.0)\n", + "Collecting distro>=1.3\n", + " Downloading distro-1.8.0-py3-none-any.whl (20 kB)\n", + "Collecting shortuuid>=0.5\n", + " Downloading shortuuid-1.0.11-py3-none-any.whl (10 kB)\n", + "Collecting rich>=12\n", + " Downloading rich-13.3.4-py3-none-any.whl (238 kB)\n", + "\u001b[K |████████████████████████████████| 238 kB 11.6 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting dvc-task<1,>=0.2.0\n", + " Downloading dvc_task-0.2.0-py3-none-any.whl (23 kB)\n", + "Collecting configobj>=5.0.6\n", + " Downloading configobj-5.0.8-py2.py3-none-any.whl (36 kB)\n", + "Collecting iterative-telemetry>=0.0.7\n", + " Downloading iterative_telemetry-0.0.8-py3-none-any.whl (10 kB)\n", + "Requirement already satisfied: six in /home/tomek/miniconda3/lib/python3.9/site-packages (from configobj>=5.0.6->dvc) (1.16.0)\n", + "Collecting dvc-objects<1,>=0.21.1\n", + " Downloading dvc_objects-0.21.2-py3-none-any.whl (37 kB)\n", + "Requirement already satisfied: attrs>=21.3.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from dvc-data<0.48,>=0.47.1->dvc) (22.2.0)\n", + "Collecting dictdiffer>=0.8.1\n", + " Using cached dictdiffer-0.9.0-py2.py3-none-any.whl (16 kB)\n", + "Collecting nanotime>=0.5.2\n", + " Using cached nanotime-0.5.2.tar.gz (3.2 kB)\n", + "Collecting diskcache>=5.2.1\n", + " Downloading diskcache-5.6.1-py3-none-any.whl (45 kB)\n", + "\u001b[K |████████████████████████████████| 45 kB 905 kB/s eta 0:00:01\n", + "\u001b[?25hCollecting sqltrie<1,>=0.3.1\n", + " Downloading sqltrie-0.3.1-py3-none-any.whl (16 kB)\n", + "Requirement already satisfied: fsspec[http] in /home/tomek/miniconda3/lib/python3.9/site-packages (from dvc-http>=2.29.0->dvc) (2023.3.0)\n", + "Collecting aiohttp-retry>=2.5.0\n", + " Downloading aiohttp_retry-2.8.3-py3-none-any.whl (9.8 kB)\n", + "Requirement already satisfied: aiohttp in /home/tomek/miniconda3/lib/python3.9/site-packages (from aiohttp-retry>=2.5.0->dvc-http>=2.29.0->dvc) (3.8.4)\n", + "Requirement already satisfied: typing-extensions>=3.7.4 in /home/tomek/miniconda3/lib/python3.9/site-packages (from dvc-objects<1,>=0.21.1->dvc-data<0.48,>=0.47.1->dvc) (4.5.0)\n", + "Collecting dulwich\n", + " Downloading dulwich-0.21.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (505 kB)\n", + "\u001b[K |████████████████████████████████| 505 kB 16.6 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting celery<6,>=5.2.0\n", + " Downloading celery-5.2.7-py3-none-any.whl (405 kB)\n", + "\u001b[K |████████████████████████████████| 405 kB 19.2 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting kombu<6,>=5.2.0\n", + " Downloading kombu-5.2.4-py3-none-any.whl (189 kB)\n", + "\u001b[K |████████████████████████████████| 189 kB 14.8 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting click-didyoumean>=0.0.3\n", + " Downloading click_didyoumean-0.3.0-py3-none-any.whl (2.7 kB)\n", + "Collecting billiard<4.0,>=3.6.4.0\n", + " Downloading billiard-3.6.4.0-py3-none-any.whl (89 kB)\n", + "\u001b[K |████████████████████████████████| 89 kB 3.8 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting vine<6.0,>=5.0.0\n", + " Downloading vine-5.0.0-py2.py3-none-any.whl (9.4 kB)\n", + "Collecting click-repl>=0.2.0\n", + " Downloading click_repl-0.2.0-py3-none-any.whl (5.2 kB)\n", + "Requirement already satisfied: click<9.0,>=8.0.3 in /home/tomek/miniconda3/lib/python3.9/site-packages (from celery<6,>=5.2.0->dvc-task<1,>=0.2.0->dvc) (8.1.3)\n", + "Collecting click-plugins>=1.1.1\n", + " Downloading click_plugins-1.1.1-py2.py3-none-any.whl (7.5 kB)\n", + "Requirement already satisfied: pytz>=2021.3 in /home/tomek/miniconda3/lib/python3.9/site-packages (from celery<6,>=5.2.0->dvc-task<1,>=0.2.0->dvc) (2022.7.1)\n", + "Requirement already satisfied: prompt-toolkit in /home/tomek/miniconda3/lib/python3.9/site-packages (from click-repl>=0.2.0->celery<6,>=5.2.0->dvc-task<1,>=0.2.0->dvc) (3.0.38)\n", + "Collecting atpublic>=2.3\n", + " Downloading atpublic-3.1.1-py3-none-any.whl (4.8 kB)\n", + "Collecting antlr4-python3-runtime==4.9.*\n", + " Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n", + "\u001b[K |████████████████████████████████| 117 kB 17.4 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting omegaconf<2.4,>=2.2\n", + " Downloading omegaconf-2.3.0-py3-none-any.whl (79 kB)\n", + "\u001b[K |████████████████████████████████| 79 kB 3.6 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting appdirs\n", + " Using cached appdirs-1.4.4-py2.py3-none-any.whl (9.6 kB)\n", + "Requirement already satisfied: filelock in /home/tomek/miniconda3/lib/python3.9/site-packages (from iterative-telemetry>=0.0.7->dvc) (3.9.1)\n", + "Collecting amqp<6.0.0,>=5.0.9\n", + " Downloading amqp-5.1.1-py3-none-any.whl (50 kB)\n", + "\u001b[K |████████████████████████████████| 50 kB 2.7 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: PyYAML>=5.1.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from omegaconf<2.4,>=2.2->hydra-core>=1.1->dvc) (6.0)\n", + "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from requests>=2.22->dvc) (1.26.9)\n", + "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from requests>=2.22->dvc) (2.0.4)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/tomek/miniconda3/lib/python3.9/site-packages (from requests>=2.22->dvc) (2022.12.7)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/tomek/miniconda3/lib/python3.9/site-packages (from requests>=2.22->dvc) (3.3)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/62/1e/a94a8d635fa3ce4cfc7f506003548d0a2447ae76fd5ca53932970fe3053f/pyasn1-0.4.8-py2.py3-none-any.whl (77kB)\n", - "\u001b[K 100% |████████████████████████████████| 81kB 2.2MB/s ta 0:00:01\n", - "\u001b[?25hCollecting pathspec<0.9.0,>=0.6.0 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/29/29/a465741a3d97ea3c17d21eaad4c64205428bde56742360876c4391f930d4/pathspec-0.8.1-py2.py3-none-any.whl\n", - "Collecting contextvars>=2.1; python_version < \"3.7\" (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/83/96/55b82d9f13763be9d672622e1b8106c85acb83edd7cc2fa5bc67cd9877e9/contextvars-2.4.tar.gz\n", - "Collecting distro>=1.3.0 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/e1/54/d08d1ad53788515392bec14d2d6e8c410bffdc127780a9a4aa8e6854d502/distro-1.7.0-py3-none-any.whl\n", - "Collecting packaging>=19.0 (from dvc)\n", - " Using cached https://files.pythonhosted.org/packages/05/8e/8de486cbd03baba4deef4142bd643a3e7bbe954a784dc1bb17142572d127/packaging-21.3-py3-none-any.whl\n", - "Collecting typing-extensions>=3.7.4 (from dvc)\n", - " Using cached https://files.pythonhosted.org/packages/45/6b/44f7f8f1e110027cf88956b59f2fad776cca7e1704396d043f89effd3a0e/typing_extensions-4.1.1-py3-none-any.whl\n", - "Collecting importlib-metadata>=1.4; python_version < \"3.8\" (from dvc)\n", - " Using cached https://files.pythonhosted.org/packages/a0/a1/b153a0a4caf7a7e3f15c2cd56c7702e2cf3d89b1b359d1f1c5e59d68f4ce/importlib_metadata-4.8.3-py3-none-any.whl\n", - "Collecting diskcache>=5.2.1 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/a1/c4/80d38cf6852ba87f8b506a91f18b3a485c668f452700689add960d7e2ecc/diskcache-5.4.0-py3-none-any.whl (44kB)\n", - "\u001b[K 100% |████████████████████████████████| 51kB 2.1MB/s ta 0:00:01\n", - "\u001b[?25hCollecting pydot>=1.2.4 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/ea/76/75b1bb82e9bad3e3d656556eaa353d8cd17c4254393b08ec9786ac8ed273/pydot-1.4.2-py2.py3-none-any.whl\n", - "Collecting funcy>=1.14 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/cd/1c/895001e29f870c4625a90af00895ef9c9f4f37b0a9b967d2ed810b7be0fc/funcy-1.17-py2.py3-none-any.whl\n", - "Collecting toml>=0.10.1 (from dvc)\n", - " Downloading https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl\n", - "Collecting requests>=2.22.0 (from dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/2d/61/08076519c80041bc0ffa1a8af0cbd3bf3e2b62af10435d269a9d0f40564d/requests-2.27.1-py2.py3-none-any.whl (63kB)\n", - "\u001b[K 100% |████████████████████████████████| 71kB 2.2MB/s ta 0:00:01\n", - "\u001b[?25hCollecting aiohttp; extra == \"http\" (from fsspec[http]>=2021.10.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/f6/3b/2e3b8a5b19cdceb532c61d83077a09afe1f120cb876fb771b0ce577cc0ea/aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1MB)\n", - "\u001b[K 100% |████████████████████████████████| 1.1MB 729kB/s ta 0:00:01\n", - "\u001b[?25hCollecting decorator<5,>=4.3 (from networkx>=2.5->dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/ed/1b/72a1821152d07cf1d8b6fce298aeb06a7eb90f4d6d41acec9861e7cc6df0/decorator-4.4.2-py2.py3-none-any.whl\n", - "Collecting atpublic (from flufl.lock<4,>=3.2->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/ab/3d/3df1468805427fedcf880da42fa26353feea3a31b5a0cc71008adcfdb816/atpublic-2.3.tar.gz\n", - "Collecting gitdb<5,>=4.0.1 (from gitpython>3->dvc)\n", - " Using cached https://files.pythonhosted.org/packages/a3/7c/5d747655049bfbf75b5fcec57c8115896cb78d6fafa84f6d3ef4c0f13a98/gitdb-4.0.9-py3-none-any.whl\n", - "Collecting importlib-resources; python_version < \"3.7\" (from tqdm<5,>=4.45.0->dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/24/1b/33e489669a94da3ef4562938cd306e8fa915e13939d7b8277cb5569cb405/importlib_resources-5.4.0-py3-none-any.whl\n", - "Collecting commonmark<0.10.0,>=0.9.0 (from rich>=10.9.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/b1/92/dfd892312d822f36c55366118b95d914e5f16de11044a27cf10a7d71bbbf/commonmark-0.9.1-py2.py3-none-any.whl (51kB)\n", - "\u001b[K 100% |████████████████████████████████| 51kB 1.6MB/s ta 0:00:01\n", - "\u001b[?25hCollecting pygments<3.0.0,>=2.6.0 (from rich>=10.9.0->dvc)\n", - " Using cached https://files.pythonhosted.org/packages/5c/8e/1d9017950034297fffa336c72e693a5b51bbf85141b24a763882cf1977b5/Pygments-2.12.0-py3-none-any.whl\n", - "Collecting six<2.0,>=1.12 (from flatten-dict<1,>=0.4.1->dvc)\n", - " Using cached https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl\n", - "Collecting urllib3>=1.24.1 (from dulwich>=0.20.23->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/ec/03/062e6444ce4baf1eac17a6a0ebfe36bb1ad05e1df0e20b110de59c278498/urllib3-1.26.9-py2.py3-none-any.whl (138kB)\n", - "\u001b[K 100% |████████████████████████████████| 143kB 1.7MB/s ta 0:00:01\n", - "\u001b[?25hCollecting certifi (from dulwich>=0.20.23->dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/11/dd/e015f3780f42dd9af62cf0107b44ea1298926627ecd70c17b0e484e95bcd/certifi-2022.5.18.1-py3-none-any.whl (155kB)\n", - "\u001b[K 100% |████████████████████████████████| 163kB 1.8MB/s ta 0:00:01\n", - "\u001b[?25hCollecting ruamel.yaml.clib>=0.2.6; platform_python_implementation == \"CPython\" and python_version < \"3.11\" (from ruamel.yaml>=0.17.11->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/2a/25/5b1dfc832ef3b83576c546d1fb3e27f136022cdd1008aab290a1e28ef220/ruamel.yaml.clib-0.2.6-cp36-cp36m-manylinux1_x86_64.whl (552kB)\n", - "\u001b[K 100% |████████████████████████████████| 552kB 1.2MB/s ta 0:00:01\n", - "\u001b[?25hCollecting python-slugify<7.0.0,>=6.0.1 (from python-benedict>=0.24.2->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/c1/35/74ab800f1108b95ff9b8e7672a01dbf1f357159e6d06c1f16e983674ff0c/python_slugify-6.1.2-py2.py3-none-any.whl\n", - "Collecting python-fsutil<1.0.0,>=0.6.0 (from python-benedict>=0.24.2->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/d3/67/3bceac53a29c2cf6a27ca9a940c7a059af13b5b4283d09368e9c0d7f91b4/python_fsutil-0.6.1-py3-none-any.whl\n", - "Collecting mailchecker<5.0.0,>=4.1.0 (from python-benedict>=0.24.2->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/1a/11/49e85a526dd3f2cb6bdf1dc519d64395bd56a88d9b2a5f6f3acd1f8c3f51/mailchecker-4.1.17.tar.gz (232kB)\n", - "\u001b[K 100% |████████████████████████████████| 235kB 2.7MB/s ta 0:00:011\n", - "\u001b[?25hCollecting pyyaml<7.0,>=6.0 (from python-benedict>=0.24.2->dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/b3/85/79b9e5b4e8d3c0ac657f4e8617713cca8408f6cdc65d2ee6554217cedff1/PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (603kB)\n", - "\u001b[K 100% |████████████████████████████████| 604kB 1.1MB/s ta 0:00:01\n", - "\u001b[?25hCollecting xmltodict<1.0.0,>=0.12.0 (from python-benedict>=0.24.2->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/94/db/fd0326e331726f07ff7f40675cd86aa804bfd2e5016c727fa761c934990e/xmltodict-0.13.0-py2.py3-none-any.whl\n", - "Collecting phonenumbers<9.0.0,>=8.12.0 (from python-benedict>=0.24.2->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/a1/c8/23c7185f589da0edf4f9831de1e718989073aa094780f8ee34e6c29b7a2e/phonenumbers-8.12.48-py2.py3-none-any.whl (2.6MB)\n", - "\u001b[K 100% |████████████████████████████████| 2.6MB 434kB/s ta 0:00:011\n", - "\u001b[?25hCollecting ftfy<7.0.0,>=6.0.0 (from python-benedict>=0.24.2->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/af/da/d215a091986e5f01b80f5145cff6f22e2dc57c6b048aab2e882a07018473/ftfy-6.0.3.tar.gz (64kB)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[K 100% |████████████████████████████████| 71kB 2.0MB/s ta 0:00:01\n", - "\u001b[?25hCollecting python-dateutil<3.0.0,>=2.8.0 (from python-benedict>=0.24.2->dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Using cached https://files.pythonhosted.org/packages/36/7a/87837f39d0296e723bb9b62bbb257d0355c7f6128853c78955f57342a56d/python_dateutil-2.8.2-py2.py3-none-any.whl\n", - "Collecting future (from grandalf==0.6->dvc)\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - "\u001b[33m Cache entry deserialization failed, entry ignored\u001b[0m\n", - " Downloading https://files.pythonhosted.org/packages/45/0b/38b06fd9b92dc2b68d58b75f900e97884c45bedd2ff83203d933cf5851c9/future-0.18.2.tar.gz (829kB)\n", - "\u001b[K 100% |████████████████████████████████| 829kB 1.5MB/s eta 0:00:01\n", - "\u001b[?25hCollecting cffi>=1.4.0 (from pygit2<1.7,>=1.5.0; python_version < \"3.7\"->dvc)\n", - " Using cached https://files.pythonhosted.org/packages/49/7b/449daf9cacfd7355cea1b4106d2be614315c29ac16567e01756167f6daab/cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl\n", - "Collecting cached-property (from pygit2<1.7,>=1.5.0; python_version < \"3.7\"->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/48/19/f2090f7dad41e225c7f2326e4cfe6fff49e57dedb5b53636c9551f86b069/cached_property-1.5.2-py2.py3-none-any.whl\n", - "Collecting immutables>=0.9 (from contextvars>=2.1; python_version < \"3.7\"->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/ff/88/9c71337193c3d24c2cf3c14d5ed05eeb502f9f21fa6117edfa9b3b43bff1/immutables-0.18-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (115kB)\n", - "\u001b[K 100% |████████████████████████████████| 122kB 2.3MB/s ta 0:00:01\n", - "\u001b[?25hCollecting zipp>=0.5 (from importlib-metadata>=1.4; python_version < \"3.8\"->dvc)\n", - " Using cached https://files.pythonhosted.org/packages/bd/df/d4a4974a3e3957fd1c1fa3082366d7fff6e428ddb55f074bf64876f8e8ad/zipp-3.6.0-py3-none-any.whl\n", - "Collecting idna<4,>=2.5; python_version >= \"3\" (from requests>=2.22.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/04/a2/d918dcd22354d8958fe113e1a3630137e0fc8b44859ade3063982eacd2a4/idna-3.3-py3-none-any.whl (61kB)\n", - "\u001b[K 100% |████████████████████████████████| 61kB 1.7MB/s ta 0:00:01\n", - "\u001b[?25hCollecting charset-normalizer~=2.0.0; python_version >= \"3\" (from requests>=2.22.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/06/b3/24afc8868eba069a7f03650ac750a778862dc34941a4bebeb58706715726/charset_normalizer-2.0.12-py3-none-any.whl\n", - "Collecting asynctest==0.13.0; python_version < \"3.8\" (from aiohttp; extra == \"http\"->fsspec[http]>=2021.10.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/e8/b6/8d17e169d577ca7678b11cd0d3ceebb0a6089a7f4a2de4b945fe4b1c86db/asynctest-0.13.0-py3-none-any.whl\n", - "Collecting yarl<2.0,>=1.0 (from aiohttp; extra == \"http\"->fsspec[http]>=2021.10.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/fa/cb/8791922f5ec97b9ebec516d062c0e113da963568ffe2c7c04d8187ab7cc3/yarl-1.7.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (270kB)\n", - "\u001b[K 100% |████████████████████████████████| 276kB 1.9MB/s ta 0:00:01\n", - "\u001b[?25hCollecting idna-ssl>=1.0; python_version < \"3.7\" (from aiohttp; extra == \"http\"->fsspec[http]>=2021.10.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/46/03/07c4894aae38b0de52b52586b24bf189bb83e4ddabfe2e2c8f2419eec6f4/idna-ssl-1.1.0.tar.gz\n", - "Collecting frozenlist>=1.1.1 (from aiohttp; extra == \"http\"->fsspec[http]>=2021.10.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/51/3f/f67395ff0090b9f2835838a1f61c3e840baac70fd65bae762095dead48b2/frozenlist-1.2.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (191kB)\n", - "\u001b[K 100% |████████████████████████████████| 194kB 1.2MB/s ta 0:00:01\n", - "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3 (from aiohttp; extra == \"http\"->fsspec[http]>=2021.10.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/d6/c1/8991e7c5385b897b8c020cdaad718c5b087a6626d1d11a23e1ea87e325a7/async_timeout-4.0.2-py3-none-any.whl\n", - "Collecting aiosignal>=1.1.2 (from aiohttp; extra == \"http\"->fsspec[http]>=2021.10.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/3b/87/fe94898f2d44a93a35d5aa74671ed28094d80753a1113d68b799fab6dc22/aiosignal-1.2.0-py3-none-any.whl\n", - "Collecting attrs>=17.3.0 (from aiohttp; extra == \"http\"->fsspec[http]>=2021.10.0->dvc)\n", - " Using cached https://files.pythonhosted.org/packages/be/be/7abce643bfdf8ca01c48afa2ddf8308c2308b0c3b239a44e57d020afa0ef/attrs-21.4.0-py2.py3-none-any.whl\n", - "Collecting multidict<7.0,>=4.5 (from aiohttp; extra == \"http\"->fsspec[http]>=2021.10.0->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/82/43/81ddfbcfbdfaeaa0624f36dcb715dc8135562377b3292e93b0315a861e92/multidict-5.2.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (159kB)\n", - "\u001b[K 100% |████████████████████████████████| 163kB 2.0MB/s ta 0:00:01\n", - "\u001b[?25hCollecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython>3->dvc)\n", - " Using cached https://files.pythonhosted.org/packages/6d/01/7caa71608bc29952ae09b0be63a539e50d2484bc37747797a66a60679856/smmap-5.0.0-py3-none-any.whl\n", - "Collecting text-unidecode>=1.3 (from python-slugify<7.0.0,>=6.0.1->python-benedict>=0.24.2->dvc)\n", - " Downloading https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl (78kB)\n", - "\u001b[K 100% |████████████████████████████████| 81kB 2.2MB/s ta 0:00:01\n", - "\u001b[?25hCollecting wcwidth (from ftfy<7.0.0,>=6.0.0->python-benedict>=0.24.2->dvc)\n", - " Using cached https://files.pythonhosted.org/packages/59/7c/e39aca596badaf1b78e8f547c807b04dae603a433d3e7a7e04d67f2ef3e5/wcwidth-0.2.5-py2.py3-none-any.whl\n", - "Collecting pycparser (from cffi>=1.4.0->pygit2<1.7,>=1.5.0; python_version < \"3.7\"->dvc)\n", - " Using cached https://files.pythonhosted.org/packages/62/d5/5f610ebe421e85889f2e55e33b7f9a6795bd982198517d912eb1c76e1a53/pycparser-2.21-py2.py3-none-any.whl\n", - "Building wheels for collected packages: nanotime, flufl.lock, psutil, configobj, pygtrie, dulwich, pygit2, contextvars, atpublic, mailchecker, ftfy, future, idna-ssl\n", - " Running setup.py bdist_wheel for nanotime ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/41/99/17/7135f635215e1f61e906295afd11f4f791cfe4ab45f3bfdca2\n", - " Running setup.py bdist_wheel for flufl.lock ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/4f/51/d7/f65a7b7f37da7594f7021b122fe677187667ad21f1171d2514\n", - " Running setup.py bdist_wheel for psutil ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/6e/94/8f/ef906811f8dcf6824a9747df0381615be48d723073fb59a317\n", - " Running setup.py bdist_wheel for configobj ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/f1/e4/16/4981ca97c2d65106b49861e0b35e2660695be7219a2d351ee0\n", - " Running setup.py bdist_wheel for pygtrie ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/db/57/91/73782136379fe419036c5ec0e4070d8b3a35f2a36bd6a94ed8\n", - " Running setup.py bdist_wheel for dulwich ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/63/28/8c/0bbff7d6e30f3fc523639b000b33aba9155152e9eb23689ba0\n", - " Running setup.py bdist_wheel for pygit2 ... \u001b[?25lerror\n", - " Complete output from command /usr/bin/python3 -u -c \"import setuptools, tokenize;__file__='/tmp/pip-build-hr8mtrcf/pygit2/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\\r\\n', '\\n');f.close();exec(compile(code, __file__, 'exec'))\" bdist_wheel -d /tmp/tmp4kyicel3pip-wheel- --python-tag cp36:\n", - " running bdist_wheel\n", - " running build\n", - " running build_py\n", - " creating build\n", - " creating build/lib.linux-x86_64-3.6\n", - " creating build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/__init__.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/_build.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/_run.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/blame.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/callbacks.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/config.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/credentials.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/errors.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/ffi.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/index.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/packbuilder.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/refspec.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/remote.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/repository.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/settings.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/submodule.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/utils.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " creating build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/attr.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/blame.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/buffer.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/callbacks.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/checkout.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/clone.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/common.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/config.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/describe.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/diff.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/errors.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/graph.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/index.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/indexer.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/merge.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/net.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/oid.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/pack.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/proxy.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/refspec.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/remote.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/repository.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/revert.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/stash.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/strarray.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/submodule.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/transport.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/types.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " running build_ext\n", - " generating cffi module 'build/temp.linux-x86_64-3.6/pygit2._libgit2.c'\n", - " creating build/temp.linux-x86_64-3.6\n", - " building 'pygit2._pygit2' extension\n", - " creating build/temp.linux-x86_64-3.6/src\n", - " x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/include -I/usr/include/python3.6m -c src/blob.c -o build/temp.linux-x86_64-3.6/src/blob.o\n", - " In file included from src/blob.c:30:0:\n", - " src/blob.h:33:10: fatal error: git2.h: No such file or directory\n", - " #include \n", - " ^~~~~~~~\n", - " compilation terminated.\n", - " error: command 'x86_64-linux-gnu-gcc' failed with exit status 1\n", - " \n", - " ----------------------------------------\n", - "\u001b[31m Failed building wheel for pygit2\u001b[0m\n", - "\u001b[?25h Running setup.py clean for pygit2\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " Running setup.py bdist_wheel for contextvars ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/a5/7d/68/1ebae2668bda2228686e3c1cf16f2c2384cea6e9334ad5f6de\n", - " Running setup.py bdist_wheel for atpublic ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/33/25/82/57d46b60a048f8e30b31f10497539498a3b826c78e2433c2d4\n", - " Running setup.py bdist_wheel for mailchecker ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/fd/e1/e7/804e77a70eac7103bdba2f4b3e1eba36840b38554a4b8152c8\n", - " Running setup.py bdist_wheel for ftfy ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/99/2c/e6/109c8a28fef7a443f67ba58df21fe1d0067ac3322e75e6b0b7\n", - " Running setup.py bdist_wheel for future ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e\n", - " Running setup.py bdist_wheel for idna-ssl ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/tomek/.cache/pip/wheels/d3/00/b3/32d613e19e08a739751dd6bf998cfed277728f8b2127ad4eb7\n", - "Successfully built nanotime flufl.lock psutil configobj pygtrie dulwich contextvars atpublic mailchecker ftfy future idna-ssl\n", - "Failed to build pygit2\n", - "Installing collected packages: setuptools, colorama, dictdiffer, ply, nanotime, asynctest, idna, multidict, typing-extensions, yarl, idna-ssl, frozenlist, async-timeout, aiosignal, charset-normalizer, attrs, aiohttp, certifi, urllib3, requests, fsspec, decorator, networkx, atpublic, flufl.lock, shtab, shortuuid, smmap, gitdb, gitpython, zipp, importlib-resources, tqdm, zc.lockfile, dataclasses, commonmark, pygments, rich, aiohttp-retry, voluptuous, psutil, six, importlib-metadata, flatten-dict, configobj, pygtrie, tabulate, dulwich, appdirs, ruamel.yaml.clib, ruamel.yaml, text-unidecode, python-slugify, python-fsutil, mailchecker, pyyaml, toml, xmltodict, phonenumbers, wcwidth, ftfy, python-dateutil, python-benedict, future, pyparsing, grandalf, dpath, pycparser, cffi, cached-property, pygit2, pyasn1, pathspec, immutables, contextvars, distro, packaging, diskcache, pydot, funcy, dvc\n", - " Running setup.py install for pygit2 ... \u001b[?25lerror\n", - " Complete output from command /usr/bin/python3 -u -c \"import setuptools, tokenize;__file__='/tmp/pip-build-hr8mtrcf/pygit2/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\\r\\n', '\\n');f.close();exec(compile(code, __file__, 'exec'))\" install --record /tmp/pip-6tml5js1-record/install-record.txt --single-version-externally-managed --compile --user --prefix=:\n", - " running install\n", - " /home/tomek/.local/lib/python3.6/site-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.\n", - " setuptools.SetuptoolsDeprecationWarning,\n", - " running build\n", - " running build_py\n", - " creating build\n", - " creating build/lib.linux-x86_64-3.6\n", - " creating build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/__init__.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/_build.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/_run.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/blame.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/callbacks.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/config.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/credentials.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/errors.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/ffi.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/index.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/packbuilder.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/refspec.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/remote.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/repository.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/settings.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/submodule.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " copying pygit2/utils.py -> build/lib.linux-x86_64-3.6/pygit2\n", - " creating build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/attr.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/blame.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/buffer.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/callbacks.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/checkout.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/clone.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/common.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/config.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/describe.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/diff.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/errors.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/graph.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/index.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/indexer.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/merge.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/net.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/oid.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/pack.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/proxy.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/refspec.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/remote.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/repository.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/revert.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/stash.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/strarray.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/submodule.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/transport.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " copying pygit2/decl/types.h -> build/lib.linux-x86_64-3.6/pygit2/decl\n", - " running build_ext\n", - " generating cffi module 'build/temp.linux-x86_64-3.6/pygit2._libgit2.c'\n", - " creating build/temp.linux-x86_64-3.6\n", - " building 'pygit2._pygit2' extension\n", - " creating build/temp.linux-x86_64-3.6/src\n", - " x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/include -I/usr/include/python3.6m -c src/blob.c -o build/temp.linux-x86_64-3.6/src/blob.o\n", - " In file included from src/blob.c:30:0:\n", - " src/blob.h:33:10: fatal error: git2.h: No such file or directory\n", - " #include \n", - " ^~~~~~~~\n", - " compilation terminated.\n", - " error: command 'x86_64-linux-gnu-gcc' failed with exit status 1\n", - " \n", - " ----------------------------------------\n", - "\u001b[31mCommand \"/usr/bin/python3 -u -c \"import setuptools, tokenize;__file__='/tmp/pip-build-hr8mtrcf/pygit2/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\\r\\n', '\\n');f.close();exec(compile(code, __file__, 'exec'))\" install --record /tmp/pip-6tml5js1-record/install-record.txt --single-version-externally-managed --compile --user --prefix=\" failed with error code 1 in /tmp/pip-build-hr8mtrcf/pygit2/\u001b[0m\n", - "\u001b[?25h" + "Collecting markdown-it-py<3.0.0,>=2.2.0\n", + " Downloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\n", + "\u001b[K |████████████████████████████████| 84 kB 1.9 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from rich>=12->dvc) (2.14.0)\n", + "Collecting mdurl~=0.1\n", + " Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n", + "Requirement already satisfied: ruamel.yaml.clib>=0.2.6 in /home/tomek/miniconda3/lib/python3.9/site-packages (from ruamel.yaml>=0.17.11->dvc) (0.2.6)\n", + "Collecting pygit2>=1.10.0\n", + " Downloading pygit2-1.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.9 MB)\n", + "\u001b[K |████████████████████████████████| 4.9 MB 13.6 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting asyncssh<3,>=2.13.1\n", + " Downloading asyncssh-2.13.1-py3-none-any.whl (348 kB)\n", + "\u001b[K |████████████████████████████████| 348 kB 38.4 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: gitpython>3 in /home/tomek/miniconda3/lib/python3.9/site-packages (from scmrepo<2,>=1.0.0->dvc) (3.1.31)\n", + "Requirement already satisfied: cryptography>=3.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from asyncssh<3,>=2.13.1->scmrepo<2,>=1.0.0->dvc) (37.0.1)\n", + "Requirement already satisfied: cffi>=1.12 in /home/tomek/miniconda3/lib/python3.9/site-packages (from cryptography>=3.1->asyncssh<3,>=2.13.1->scmrepo<2,>=1.0.0->dvc) (1.15.0)\n", + "Requirement already satisfied: pycparser in /home/tomek/miniconda3/lib/python3.9/site-packages (from cffi>=1.12->cryptography>=3.1->asyncssh<3,>=2.13.1->scmrepo<2,>=1.0.0->dvc) (2.21)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from gitpython>3->scmrepo<2,>=1.0.0->dvc) (4.0.10)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from gitdb<5,>=4.0.1->gitpython>3->scmrepo<2,>=1.0.0->dvc) (5.0.0)\n", + "Collecting orjson\n", + " Downloading orjson-3.8.10-cp39-cp39-manylinux_2_28_x86_64.whl (140 kB)\n", + "\u001b[K |████████████████████████████████| 140 kB 39.5 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: setuptools in /home/tomek/miniconda3/lib/python3.9/site-packages (from zc.lockfile>=1.2.1->dvc) (61.2.0)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/tomek/miniconda3/lib/python3.9/site-packages (from aiohttp->aiohttp-retry>=2.5.0->dvc-http>=2.29.0->dvc) (4.0.2)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /home/tomek/miniconda3/lib/python3.9/site-packages (from aiohttp->aiohttp-retry>=2.5.0->dvc-http>=2.29.0->dvc) (6.0.4)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /home/tomek/miniconda3/lib/python3.9/site-packages (from aiohttp->aiohttp-retry>=2.5.0->dvc-http>=2.29.0->dvc) (1.8.2)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /home/tomek/miniconda3/lib/python3.9/site-packages (from aiohttp->aiohttp-retry>=2.5.0->dvc-http>=2.29.0->dvc) (1.3.1)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /home/tomek/miniconda3/lib/python3.9/site-packages (from aiohttp->aiohttp-retry>=2.5.0->dvc-http>=2.29.0->dvc) (1.3.3)\n", + "Requirement already satisfied: wcwidth in /home/tomek/miniconda3/lib/python3.9/site-packages (from prompt-toolkit->click-repl>=0.2.0->celery<6,>=5.2.0->dvc-task<1,>=0.2.0->dvc) (0.2.6)\n", + "Building wheels for collected packages: antlr4-python3-runtime, nanotime\n", + " Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144575 sha256=94691fc7a4109d606872ddee3ae9575c3c9f9f945643a27b5514fce3176c552a\n", + " Stored in directory: /home/tomek/.cache/pip/wheels/23/cf/80/f3efa822e6ab23277902ee9165fe772eeb1dfb8014f359020a\n", + " Building wheel for nanotime (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for nanotime: filename=nanotime-0.5.2-py3-none-any.whl size=2441 sha256=42933d16d8f6362832282dea6b0b44f2bdd41b0eb0d68de121660a8a0db1f96c\n", + " Stored in directory: /home/tomek/.cache/pip/wheels/ee/1f/7c/610bdb7d5541b98d9743c5953e32681ef35dd54fadddd347e8\n", + "Successfully built antlr4-python3-runtime nanotime\n", + "Installing collected packages: vine, amqp, shortuuid, pygtrie, orjson, mdurl, kombu, funcy, click-repl, click-plugins, click-didyoumean, billiard, antlr4-python3-runtime, voluptuous, sqltrie, pygit2, psutil, pathspec, omegaconf, nanotime, markdown-it-py, dvc-objects, dulwich, distro, diskcache, dictdiffer, celery, atpublic, asyncssh, appdirs, aiohttp-retry, zc.lockfile, tomlkit, shtab, scmrepo, rich, pydot, networkx, iterative-telemetry, hydra-core, grandalf, flufl.lock, flatten-dict, dvc-task, dvc-studio-client, dvc-render, dvc-http, dvc-data, dpath, configobj, dvc\n", + "Successfully installed aiohttp-retry-2.8.3 amqp-5.1.1 antlr4-python3-runtime-4.9.3 appdirs-1.4.4 asyncssh-2.13.1 atpublic-3.1.1 billiard-3.6.4.0 celery-5.2.7 click-didyoumean-0.3.0 click-plugins-1.1.1 click-repl-0.2.0 configobj-5.0.8 dictdiffer-0.9.0 diskcache-5.6.1 distro-1.8.0 dpath-2.1.5 dulwich-0.21.3 dvc-2.55.0 dvc-data-0.47.2 dvc-http-2.30.2 dvc-objects-0.21.2 dvc-render-0.3.1 dvc-studio-client-0.8.0 dvc-task-0.2.0 flatten-dict-0.4.2 flufl.lock-7.1.1 funcy-2.0 grandalf-0.8 hydra-core-1.3.2 iterative-telemetry-0.0.8 kombu-5.2.4 markdown-it-py-2.2.0 mdurl-0.1.2 nanotime-0.5.2 networkx-3.1 omegaconf-2.3.0 orjson-3.8.10 pathspec-0.11.1 psutil-5.9.5 pydot-1.4.2 pygit2-1.12.0 pygtrie-2.5.0 rich-13.3.4 scmrepo-1.0.2 shortuuid-1.0.11 shtab-1.6.1 sqltrie-0.3.1 tomlkit-0.11.7 vine-5.0.0 voluptuous-0.13.1 zc.lockfile-3.0.post1\n" ] } ], @@ -519,25 +313,26 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "aae59ec2", + "execution_count": 7, + "id": "4d94e912", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [], "source": [ - "!mkdir -p IUM_10/sample-ml-project" + "!rm -r -f IUM_10/sample-ml-project-2023\n", + "!mkdir -p IUM_10/sample-ml-project-2023" ] }, { "cell_type": "code", - "execution_count": 1, - "id": "1e522a93", + "execution_count": 8, + "id": "aae59ec2", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -545,13 +340,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "/home/tomek/repos/aitech-ium/IUM_10/sample-ml-project\n" + "/home/tomek/repos/aitech-ium/IUM_10/sample-ml-project-2023/IUM_10/sample-ml-project-2023\n" ] } ], "source": [ "#Jupyter notebook magic https://ipython.readthedocs.io/en/stable/interactive/magics.html#magic-cd\n", - "%cd \"IUM_10/sample-ml-project\"" + "%cd \"IUM_10/sample-ml-project-2023\"" ] }, { @@ -568,11 +363,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 9, "id": "c13c525b", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -580,7 +375,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Reinitialized existing Git repository in /home/tomek/repos/aitech-ium/IUM_10/sample-ml-project/.git/\r\n" + "Initialized empty Git repository in /home/tomek/repos/aitech-ium/IUM_10/sample-ml-project-2023/IUM_10/sample-ml-project-2023/.git/\r\n" ] } ], @@ -602,11 +397,11 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 10, "id": "44f28226", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -614,7 +409,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[31mERROR\u001b[39m: failed to initiate DVC - '.dvc' exists. Use `-f` to force.\n", + "Initialized DVC repository.\n", + "\n", + "You can now commit the changes to git.\n", + "\n", + "\u001b[31m+---------------------------------------------------------------------+\n", + "\u001b[0m\u001b[31m|\u001b[0m \u001b[31m|\u001b[0m\n", + "\u001b[31m|\u001b[0m DVC has enabled anonymous aggregate usage analytics. \u001b[31m|\u001b[0m\n", + "\u001b[31m|\u001b[0m Read the analytics documentation (and how to opt-out) here: \u001b[31m|\u001b[0m\n", + "\u001b[31m|\u001b[0m <\u001b[36mhttps://dvc.org/doc/user-guide/analytics\u001b[39m> \u001b[31m|\u001b[0m\n", + "\u001b[31m|\u001b[0m \u001b[31m|\u001b[0m\n", + "\u001b[31m+---------------------------------------------------------------------+\n", + "\u001b[0m\n", + "\u001b[33mWhat's next?\u001b[39m\n", + "\u001b[33m------------\u001b[39m\n", + "- Check out the documentation: <\u001b[36mhttps://dvc.org/doc\u001b[39m>\n", + "- Get help and share ideas: <\u001b[36mhttps://dvc.org/chat\u001b[39m>\n", + "- Star us on GitHub: <\u001b[36mhttps://github.com/iterative/dvc\u001b[39m>\n", "\u001b[0m" ] } @@ -638,11 +449,11 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 11, "id": "d1aefe16", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -650,10 +461,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "On branch master\r\n", + "On branch main\r\n", + "\r\n", + "No commits yet\r\n", + "\r\n", "Changes to be committed:\r\n", - " (use \"git restore --staged ...\" to unstage)\r\n", - "\t\u001b[32mmodified: .dvc/config\u001b[m\r\n", + " (use \"git rm --cached ...\" to unstage)\r\n", + "\t\u001b[32mnew file: .dvc/.gitignore\u001b[m\r\n", + "\t\u001b[32mnew file: .dvc/config\u001b[m\r\n", + "\t\u001b[32mnew file: .dvcignore\u001b[m\r\n", "\r\n" ] } @@ -662,6 +478,21 @@ "!git status" ] }, + { + "cell_type": "markdown", + "id": "b16a62e6", + "metadata": { + "slideshow": { + "slide_type": "fragment" + } + }, + "source": [ + "- `.dvc/config` - główny plik konfiguracyjny dvc\n", + "- `.dvc/config.local` - nadpisuje wartości z `config`, do lokalnych zmian nie commitowanych do repo\n", + "- `.dvc/.gitignore` - pliki dvc, które nie mają znaleźć się w repo\n", + "- `.dvcignore` - dvc pomija pliki zdefiniowane w tym pliku (np. aby poprawić wydajność)" + ] + }, { "cell_type": "markdown", "id": "72e0a272", @@ -676,12 +507,12 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 12, "id": "59780e99", "metadata": { "scrolled": true, "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -689,7 +520,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[master (root-commit) d00d0ac] Initial commit\r\n", + "[main (root-commit) 6b03a40] Initial commit\r\n", " 3 files changed, 6 insertions(+)\r\n", " create mode 100644 .dvc/.gitignore\r\n", " create mode 100644 .dvc/config\r\n", @@ -701,39 +532,6 @@ "!git commit -m \"Initial commit\"" ] }, - { - "cell_type": "markdown", - "id": "3d4ce1cb", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "## Śledzenie plików za pomocą DVC\n", - " - dużymi plikami, takimi jak plikami z danymi wejściowymi czy plikami modeli, trudno zarządza się za pomocą gita, ze względu na problemy z:\n", - " - wydajnością\n", - " - przestrzenią w repozytorium\n", - " - ograniczenia ze strony serwisu (np. [limit 100 MB na plik w Github](https://docs.github.com/en/repositories/working-with-files/managing-large-files/about-large-files-on-github))\n", - " - Git posiada rozszerzenie [lfs(Large File Storage)](https://git-lfs.github.com/), które stanowi pewne rozwiązanie tego problemu. Same pliki przechowywane są na specjalnym zdalnym serwerze, w repozytorium przechowywane są jedynie odnośniki do tych plików i pewne metadane" - ] - }, - { - "cell_type": "markdown", - "id": "dd8e529b", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - " - DVC proponuje podobne podejście, ale:\n", - " - pliki mogą być przechowywane na niemal dowolnym serwerze, również lokalnie\n", - " - brak limitu wielkości plików (w Git-LFS na Github [limit 2GB](https://docs.github.com/en/repositories/working-with-files/managing-large-files/about-git-large-file-storage))\n", - " - DVC zapewnia dodatkowe narzędzie umożliwiające śledzenie plików i ich powiązań z wynikami eksperymentów\n", - " - więcej, patrz [tutaj](https://dvc.org/doc/user-guide/related-technologies)" - ] - }, { "cell_type": "markdown", "id": "a8861abe", @@ -748,11 +546,11 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 13, "id": "f05ece1b", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -760,9 +558,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Downloading iris.zip to /home/tomek/repos/aitech-ium/IUM_10/sample-ml-project\n", + "Downloading iris.zip to /home/tomek/repos/aitech-ium/IUM_10/sample-ml-project-2023/IUM_10/sample-ml-project-2023\n", " 0%| | 0.00/3.60k [00:00...\" to unstage)\r\n", - "\t\u001b[32mmodified: .dvc/config\u001b[m\r\n", + "On branch main\r\n", + "Untracked files:\r\n", + " (use \"git add ...\" to include in what will be committed)\r\n", + "\t\u001b[31mdata/.gitignore\u001b[m\r\n", + "\t\u001b[31mdata/Iris.csv.dvc\u001b[m\r\n", "\r\n", - "Changes not staged for commit:\r\n", - " (use \"git add ...\" to update what will be committed)\r\n", - " (use \"git restore ...\" to discard changes in working directory)\r\n", - "\t\u001b[31mmodified: data/Iris.csv.dvc\u001b[m\r\n", - "\r\n" + "nothing added to commit but untracked files present (use \"git add\" to track)\r\n" ] } ], @@ -893,11 +681,11 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 17, "id": "460c4a17", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [], @@ -907,11 +695,11 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 18, "id": "80644077", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -919,8 +707,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "[master b37a8d8] Dodano dane IRIS (DVC)\r\n", - " 2 files changed, 2 insertions(+), 6 deletions(-)\r\n" + "[main 812cb53] Dodano dane IRIS (DVC)\r\n", + " 2 files changed, 5 insertions(+)\r\n", + " create mode 100644 data/.gitignore\r\n", + " create mode 100644 data/Iris.csv.dvc\r\n" ] } ], @@ -946,7 +736,7 @@ "id": "8cb2ba7c", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [], @@ -972,11 +762,11 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 25, "id": "1d471f3a", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -995,11 +785,11 @@ }, { "cell_type": "code", - "execution_count": 17, - "id": "f86a5b55", + "execution_count": 33, + "id": "32531aa8", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1007,14 +797,47 @@ "name": "stdout", "output_type": "stream", "text": [ - "total 8\r\n", - "-rw-r--r-- 1 tomek tomek 5107 May 29 09:19 Iris.csv\r\n", - "-rw-r--r-- 1 tomek tomek 76 May 29 09:19 Iris.csv.dvc\r\n" + "Id,SepalLengthCm,SepalWidthCm,PetalLengthCm,PetalWidthCm,Species\r\n", + "1,5.1,3.5,1.4,0.2,Iris-setosa\r\n", + "2,4.9,3.0,1.4,0.2,Iris-setosa\r\n" ] } ], "source": [ - "!ls -l ./data" + "!head -n 3 .dvc/cache/71/7820ef0af287ff346c5cabfb4c612c" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "2396c762", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Enumerating objects: 11, done.\n", + "Counting objects: 100% (11/11), done.\n", + "Delta compression using up to 4 threads\n", + "Compressing objects: 100% (8/8), done.\n", + "Writing objects: 100% (11/11), 889 bytes | 889.00 KiB/s, done.\n", + "Total 11 (delta 1), reused 0 (delta 0), pack-reused 0\n", + "remote: \n", + "remote: Create a new pull request for 'main':\u001b[K\n", + "remote: https://git.wmi.amu.edu.pl/tzietkiewicz/sample-ml-project/compare/master...main\u001b[K\n", + "remote: \n", + "remote: . Processing 1 references\u001b[K\n", + "remote: Processed 1 references in total\u001b[K\n", + "To git.wmi.amu.edu.pl:tzietkiewicz/sample-ml-project.git\n", + " * [new branch] main -> main\n", + "Branch 'main' set up to track remote branch 'main' from 'origin'.\n" + ] + } + ], + "source": [ + "!git remote add origin git@git.wmi.amu.edu.pl:tzietkiewicz/sample-ml-project.git\n", + "!git push --set-upstream origin main" ] }, { @@ -1057,13 +880,25 @@ " - local remote" ] }, + { + "cell_type": "markdown", + "id": "507e3a09", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "#### Dodawanie remote typu local" + ] + }, { "cell_type": "code", "execution_count": 71, "id": "a16f2bfa", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1086,7 +921,7 @@ "id": "9c3deeaf", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1105,11 +940,11 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 34, "id": "899eac7d", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1117,7 +952,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "On branch master\r\n", + "On branch main\r\n", "nothing to commit, working tree clean\r\n" ] } @@ -1146,7 +981,7 @@ "id": "c7f24f75", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1173,7 +1008,7 @@ "id": "8a355575", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1226,7 +1061,7 @@ "id": "dde39796", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [], @@ -1240,7 +1075,7 @@ "id": "7f14ec60", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1263,7 +1098,7 @@ "id": "8a841039", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1642,13 +1477,13 @@ } }, "source": [ - "## Zadania [15pkt]\n", + "## Zadania [10+5 pkt]\n", "1. Zainicjalizuj repozytorium DVC wewnątrz Twojego repozytorium z projektem [1pkt]\n", "2. Dodaj plik(i) z danymi w Twoim projekcie do DVC [1pkt]\n", "3. Skonfiguruj remote (dane do konfiguracji podane poniżej) [3pkt]\n", "4. Stwórz/zdefiniuj i dodaj do repozytorium plik `dvc.yaml` opisujący kroki wykonywane w Twoim projekcie. Wydziel przynajmniej 2 kroki (np. przygotowanie danych/trenowanie) powiązane ze sobą za pomocą zależności (skorzystaj z \n", - "materiałów \"Getting started\", link powyżej) [6pkt]\n", - "5. Stwórz projekt na Jenkinsie (`s1233456-dvc`), w którym sklonujesz repozytorium, ściągniesz pliki dvc (za pomocą `dvc pull`) i uruchomisz pipeline (za pomocą `dvc reproduce`) [4pkt]" + "materiałów \"Getting started\", link powyżej) [5pkt (opcjonalne)]\n", + "5. Stwórz projekt na Jenkinsie (`s1233456-dvc`), w którym sklonujesz repozytorium, ściągniesz pliki dvc (za pomocą `dvc pull`) i uruchomisz pipeline (za pomocą `dvc reproduce`) [5pkt]" ] }, { @@ -1765,13 +1600,25 @@ "conda install -c conda-forge dvc-ssh" ] }, + { + "cell_type": "markdown", + "id": "04c41da0", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Dodajemy remote:" + ] + }, { "cell_type": "code", "execution_count": 17, "id": "e9a04876", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1794,7 +1641,7 @@ "id": "e3f27bbb", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1814,7 +1661,11 @@ { "cell_type": "markdown", "id": "c92edd7b", - "metadata": {}, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, "source": [ "Zapisujemy hasło:" ] @@ -1825,7 +1676,7 @@ "id": "5b2fa175", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1841,13 +1692,25 @@ "!dvc remote modify --local ium_ssh_remote password IUM@2021" ] }, + { + "cell_type": "markdown", + "id": "8b83049b", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Pushujemy do skonfigurowanego remote:" + ] + }, { "cell_type": "code", "execution_count": 20, "id": "ea6e16fa", "metadata": { "slideshow": { - "slide_type": "slide" + "slide_type": "fragment" } }, "outputs": [ @@ -1883,8 +1746,8 @@ "\n", "Takie dane dla użytkownika ium-sftp zostały stworzone na Jenkinsie:\n", "\n", - " - typu ssh key: https://tzietkiewicz.vm.wmi.amu.edu.pl:8080/credentials/store/system/domain/_/credential/48ac7004-216e-4260-abba-1fe5db753e18/\n", - " - typu \"secret text\" - zawierający hasło użytkownika ium-shftp: https://tzietkiewicz.vm.wmi.amu.edu.pl:8080/credentials/store/system/domain/_/credential/ium-sftp-password/\n", + " - typu ssh key: https://tzietkiewicz.vm.wmi.amu.edu.pl:8081/credentials/store/system/domain/_/credential/48ac7004-216e-4260-abba-1fe5db753e18/\n", + " - typu \"secret text\" - zawierający hasło użytkownika ium-shftp: https://tzietkiewicz.vm.wmi.amu.edu.pl:8081/credentials/store/system/domain/_/credential/ium-sftp-password/\n", "\n", "Opis używania \"Credentials\" w Jenkinsfile: https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#for-other-credential-types\n", "\n", @@ -1908,67 +1771,10 @@ " }\n", "```\n", "\n", - "Przykład kongiguracji: \n", - " - https://tzietkiewicz.vm.wmi.amu.edu.pl:8080/job/docker-test-mount/ \n", + "Przykład konfiguracji: \n", + " - https://tzietkiewicz.vm.wmi.amu.edu.pl:8081/job/docker-test-mount/ \n", " - https://git.wmi.amu.edu.pl/tzietkiewicz/ium-helloworld" ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "c6fd9e8d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Everything is up to date. \n", - "\u001b[0m" - ] - } - ], - "source": [ - "!dvc pull" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "3efbcba0", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "error: remote origin already exists.\n", - "Enumerating objects: 24, done.\n", - "Counting objects: 100% (24/24), done.\n", - "Delta compression using up to 4 threads\n", - "Compressing objects: 100% (21/21), done.\n", - "Writing objects: 100% (24/24), 2.10 KiB | 715.00 KiB/s, done.\n", - "Total 24 (delta 4), reused 0 (delta 0), pack-reused 0\n", - "remote: . Processing 1 references\u001b[K\n", - "remote: Processed 1 references in total\u001b[K\n", - "To git.wmi.amu.edu.pl:tzietkiewicz/sample-ml-project.git\n", - " * [new branch] master -> master\n", - "Branch 'master' set up to track remote branch 'master' from 'origin'.\n" - ] - } - ], - "source": [ - "!git remote add origin git@git.wmi.amu.edu.pl:tzietkiewicz/sample-ml-project.git\n", - "!git push --set-upstream origin master" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "67896105", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -1976,7 +1782,7 @@ "celltoolbar": "Slideshow", "email": "tomasz.zietkiewicz@amu.edu.pl", "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -1991,7 +1797,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "slideshow": { "slide_type": "slide"