From 78aef5d79e6b91fc250757f52bdf9e803b284863 Mon Sep 17 00:00:00 2001 From: Tomasz Date: Sun, 29 May 2022 09:51:07 +0200 Subject: [PATCH] Updated DVC remote config --- IUM_10.DVC.ipynb | 224 ++++++++++++++++++++++++++++++----------------- 1 file changed, 142 insertions(+), 82 deletions(-) diff --git a/IUM_10.DVC.ipynb b/IUM_10.DVC.ipynb index 254ac49..8e81d10 100644 --- a/IUM_10.DVC.ipynb +++ b/IUM_10.DVC.ipynb @@ -519,7 +519,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 1, "id": "aae59ec2", "metadata": { "slideshow": { @@ -533,7 +533,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "1e522a93", "metadata": { "slideshow": { @@ -545,7 +545,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "/home/tomek/AITech/repo/aitech-ium-private/IUM_10/sample-ml-project\n" + "/home/tomek/repos/aitech-ium/IUM_10/sample-ml-project\n" ] } ], @@ -568,7 +568,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 2, "id": "c13c525b", "metadata": { "slideshow": { @@ -580,7 +580,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Initialized empty Git repository in /home/tomek/AITech/repo/aitech-ium-private/IUM_10/sample-ml-project/.git/\r\n" + "Reinitialized existing Git repository in /home/tomek/repos/aitech-ium/IUM_10/sample-ml-project/.git/\r\n" ] } ], @@ -602,7 +602,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 75, "id": "44f28226", "metadata": { "slideshow": { @@ -654,7 +654,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 4, "id": "d1aefe16", "metadata": { "slideshow": { @@ -674,12 +674,6 @@ " (use \"git rm --cached ...\" to unstage)\r\n", "\t\u001b[32mnew file: .dvc/.gitignore\u001b[m\r\n", "\t\u001b[32mnew file: .dvc/config\u001b[m\r\n", - "\t\u001b[32mnew file: .dvc/plots/confusion.json\u001b[m\r\n", - "\t\u001b[32mnew file: .dvc/plots/confusion_normalized.json\u001b[m\r\n", - "\t\u001b[32mnew file: .dvc/plots/default.json\u001b[m\r\n", - "\t\u001b[32mnew file: .dvc/plots/linear.json\u001b[m\r\n", - "\t\u001b[32mnew file: .dvc/plots/scatter.json\u001b[m\r\n", - "\t\u001b[32mnew file: .dvc/plots/smooth.json\u001b[m\r\n", "\t\u001b[32mnew file: .dvcignore\u001b[m\r\n", "\r\n" ] @@ -703,9 +697,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "59780e99", "metadata": { + "scrolled": true, "slideshow": { "slide_type": "slide" } @@ -715,8 +710,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "On branch master\r\n", - "nothing to commit, working tree clean\r\n" + "[master (root-commit) d00d0ac] Initial commit\r\n", + " 3 files changed, 6 insertions(+)\r\n", + " create mode 100644 .dvc/.gitignore\r\n", + " create mode 100644 .dvc/config\r\n", + " create mode 100644 .dvcignore\r\n" ] } ], @@ -726,7 +724,7 @@ }, { "cell_type": "markdown", - "id": "dd8e529b", + "id": "3d4ce1cb", "metadata": { "slideshow": { "slide_type": "slide" @@ -738,7 +736,18 @@ " - wydajnością\n", " - przestrzenią w repozytorium\n", " - ograniczenia ze strony serwisu (np. [limit 100 MB na plik w Github](https://docs.github.com/en/repositories/working-with-files/managing-large-files/about-large-files-on-github))\n", - " - Git posiada rozszerzenie [lfs(Large File Storage)](https://git-lfs.github.com/), które stanowi pewne rozwiązanie tego problemu. Same pliki przechowywane są na specjalnym zdalnym serwerze, w repozytorium przechowywane są jedynie odnośniki do tych plików i pewne metadane\n", + " - Git posiada rozszerzenie [lfs(Large File Storage)](https://git-lfs.github.com/), które stanowi pewne rozwiązanie tego problemu. Same pliki przechowywane są na specjalnym zdalnym serwerze, w repozytorium przechowywane są jedynie odnośniki do tych plików i pewne metadane" + ] + }, + { + "cell_type": "markdown", + "id": "dd8e529b", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ " - DVC proponuje podobne podejście, ale:\n", " - pliki mogą być przechowywane na niemal dowolnym serwerze, również lokalnie\n", " - brak limitu wielkości plików (w Git-LFS na Github [limit 2GB](https://docs.github.com/en/repositories/working-with-files/managing-large-files/about-git-large-file-storage))\n", @@ -760,7 +769,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 9, "id": "f05ece1b", "metadata": { "slideshow": { @@ -772,9 +781,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "Downloading iris.zip to /home/tomek/AITech/repo/aitech-ium-private/IUM_10/sample-ml-project\n", + "Warning: Your Kaggle API key is readable by other users on this system! To fix this, you can run 'chmod 600 /home/tomek/.kaggle/kaggle.json'\n", + "Downloading iris.zip to /home/tomek/repos/aitech-ium/IUM_10/sample-ml-project\n", " 0%| | 0.00/3.60k [00:00...\" to include in what will be committed)\r\n", "\t\u001b[31mdata/.gitignore\u001b[m\r\n", "\t\u001b[31mdata/Iris.csv.dvc\u001b[m\r\n", - "\t\u001b[31miris.zip\u001b[m\r\n", "\r\n", "nothing added to commit but untracked files present (use \"git add\" to track)\r\n" ] @@ -887,7 +912,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 12, "id": "460c4a17", "metadata": { "slideshow": { @@ -901,7 +926,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 13, "id": "80644077", "metadata": { "slideshow": { @@ -913,10 +938,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "[master cc0821a] Dodano dane IRIS (DVC)\r\n", - " 2 files changed, 5 insertions(+)\r\n", - " create mode 100644 data/.gitignore\r\n", - " create mode 100644 data/Iris.csv.dvc\r\n" + "[master 67214ea] Dodano dane IRIS (DVC)\n", + " 2 files changed, 5 insertions(+)\n", + " create mode 100644 data/.gitignore\n", + " create mode 100644 data/Iris.csv.dvc\n" ] } ], @@ -968,7 +993,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 16, "id": "1d471f3a", "metadata": { "slideshow": { @@ -981,7 +1006,7 @@ "output_type": "stream", "text": [ "total 8\r\n", - "-r--r--r-- 1 tomek tomek 5107 wrz 19 2019 7820ef0af287ff346c5cabfb4c612c\r\n" + "-r--r--r-- 1 tomek tomek 5107 Sep 19 2019 7820ef0af287ff346c5cabfb4c612c\r\n" ] } ], @@ -991,8 +1016,8 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "9666e0ae", + "execution_count": 17, + "id": "f86a5b55", "metadata": { "slideshow": { "slide_type": "slide" @@ -1003,7 +1028,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "ls: cannot access './data': No such file or directory\r\n" + "total 8\r\n", + "-rw-r--r-- 1 tomek tomek 5107 May 29 09:19 Iris.csv\r\n", + "-rw-r--r-- 1 tomek tomek 76 May 29 09:19 Iris.csv.dvc\r\n" ] } ], @@ -1029,8 +1056,12 @@ }, { "cell_type": "markdown", - "id": "5e47dbd3", - "metadata": {}, + "id": "53429521", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, "source": [ "Obsługiwane typy zdalnych lokalizacji (remotes): https://dvc.org/doc/command-reference/remote/add#supported-storage-types\n", " - Amazon S3\n", @@ -1049,8 +1080,8 @@ }, { "cell_type": "code", - "execution_count": 28, - "id": "731f6ea4", + "execution_count": 71, + "id": "a16f2bfa", "metadata": { "slideshow": { "slide_type": "slide" @@ -1061,7 +1092,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Setting 'my_local_remote' as a default remote.\r\n", + "Setting 'my_local_remote' as a default remote.\n", "\u001b[0m" ] } @@ -1072,7 +1103,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 25, "id": "9c3deeaf", "metadata": { "slideshow": { @@ -1085,12 +1116,7 @@ "output_type": "stream", "text": [ "On branch master\r\n", - "Changes not staged for commit:\r\n", - " (use \"git add ...\" to update what will be committed)\r\n", - " (use \"git restore ...\" to discard changes in working directory)\r\n", - "\t\u001b[31mmodified: .dvc/config\u001b[m\r\n", - "\r\n", - "no changes added to commit (use \"git add\" and/or \"git commit -a\")\r\n" + "nothing to commit, working tree clean\r\n" ] } ], @@ -1100,7 +1126,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 26, "id": "899eac7d", "metadata": { "slideshow": { @@ -1112,8 +1138,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "[master 3ff62b6] Added DVC remote\r\n", - " 1 file changed, 4 insertions(+)\r\n" + "On branch master\r\n", + "nothing to commit, working tree clean\r\n" ] } ], @@ -1137,21 +1163,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "c7f24f75", "metadata": { "slideshow": { "slide_type": "slide" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 0% Transferring| |0/1 [00:00