From d7b29ebaf54a5cfaf0d5046fdc4b9a1a36e8795b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomasz=20Zi=C4=99tkiewicz?= <tomasz.zietkiewicz@amu.edu.pl>
Date: Fri, 4 Jun 2021 15:52:21 +0200
Subject: [PATCH] Dodano opis konfiguracji remove DVC

---
 IUM_10.DVC.ipynb   | 270 ++++++++++++++++++++++++++++++++++++++++-----
 IUM_12.Praca.ipynb |   2 +-
 2 files changed, 243 insertions(+), 29 deletions(-)

diff --git a/IUM_10.DVC.ipynb b/IUM_10.DVC.ipynb
index 1c080cd..4674704 100644
--- a/IUM_10.DVC.ipynb
+++ b/IUM_10.DVC.ipynb
@@ -303,7 +303,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 2,
    "id": "1e522a93",
    "metadata": {
     "slideshow": {
@@ -864,26 +864,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
-   "id": "8ecf3091",
-   "metadata": {
-    "slideshow": {
-     "slide_type": "slide"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "  0% Uploading|                                      |0/1 [00:00<?,     ?file/s]\n",
-      "!\u001b[A\n",
-      "  0%|          |data/Iris.csv                  0.00/5.11k [00:00<?,       ?it/s]\u001b[A\n",
-      "1 file pushed                                                                   \u001b[A\n",
-      "\u001b[0m"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "id": "c7f24f75",
+   "metadata": {},
+   "outputs": [],
    "source": [
     "!dvc push"
    ]
@@ -1196,13 +1180,7 @@
       " 52%|█████▏    |get-started/data.xml      18.6M/36.1M [00:13<00:03,    4.84MB/s]\u001b[A\n",
       " 53%|█████▎    |get-started/data.xml      19.2M/36.1M [00:13<00:03,    5.05MB/s]\u001b[A\n",
       " 55%|█████▍    |get-started/data.xml      19.8M/36.1M [00:13<00:03,    5.16MB/s]\u001b[A\n",
-      " 57%|█████▋    |get-started/data.xml      20.4M/36.1M [00:13<00:03,    5.35MB/s]\u001b[A\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
+      " 57%|█████▋    |get-started/data.xml      20.4M/36.1M [00:13<00:03,    5.35MB/s]\u001b[A\n",
       " 58%|█████▊    |get-started/data.xml      21.1M/36.1M [00:13<00:02,    5.49MB/s]\u001b[A\n",
       " 60%|██████    |get-started/data.xml      21.8M/36.1M [00:13<00:02,    5.66MB/s]\u001b[A\n",
       " 62%|██████▏   |get-started/data.xml      22.4M/36.1M [00:13<00:02,    5.83MB/s]\u001b[A\n",
@@ -1360,6 +1338,242 @@
     "4. Stwórz/zdefiniuj i dodaj do repozytorium plik `dvc.yaml` opisujący kroki wykonywane w Twoim projekcie. Wydziel przynajmniej 2 kroki (np. przygotowanie danych/trenowanie) powiązane ze sobą za pomocą zależności (skorzystaj z materiałów \"Getting started\", link powyżej) [6pkt]\n",
     "5. Stwórz projekt na Jenkinsie (`s1233456-dvc`), w którym sklonujesz repozytorium, ściągniesz pliki dvc (za pomocą `dvc pull`) i uruchomisz pipeline (za pomocą `dvc reproduce`) [6pkt]"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2f5a8590",
+   "metadata": {},
+   "source": [
+    "## SSH remote"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7ad575c8",
+   "metadata": {},
+   "source": [
+    "### Lokalnie:"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "82a61107",
+   "metadata": {},
+   "source": [
+    "Będziemy potrzebować zależności :\n",
+    "  \n",
+    "  `conda install dvc-ssh` \n",
+    "\n",
+    "albo\n",
+    "\n",
+    "`pip install dvc[ssh] paramiko`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "c48c5b8e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting package metadata (current_repodata.json): done\n",
+      "Solving environment: done\n",
+      "\n",
+      "## Package Plan ##\n",
+      "\n",
+      "  environment location: /home/tomek/miniconda3\n",
+      "\n",
+      "  added / updated specs:\n",
+      "    - dvc-ssh\n",
+      "\n",
+      "\n",
+      "The following packages will be downloaded:\n",
+      "\n",
+      "    package                    |            build\n",
+      "    ---------------------------|-----------------\n",
+      "    bcrypt-3.2.0               |   py39h3811e60_1          44 KB  conda-forge\n",
+      "    ca-certificates-2021.5.30  |       ha878542_0         136 KB  conda-forge\n",
+      "    certifi-2021.5.30          |   py39hf3d152e_0         141 KB  conda-forge\n",
+      "    dvc-2.3.0                  |   py39hf3d152e_0         542 KB  conda-forge\n",
+      "    dvc-ssh-2.3.0              |   py39hf3d152e_0           9 KB  conda-forge\n",
+      "    fsspec-2021.5.0            |     pyhd8ed1ab_0          77 KB  conda-forge\n",
+      "    invoke-1.5.0               |     pyhd3deb0d_0         137 KB  conda-forge\n",
+      "    paramiko-2.7.2             |     pyh9f0ad1d_0         135 KB  conda-forge\n",
+      "    pynacl-1.4.0               |   py39h3811e60_2         1.3 MB  conda-forge\n",
+      "    ------------------------------------------------------------\n",
+      "                                           Total:         2.5 MB\n",
+      "\n",
+      "The following NEW packages will be INSTALLED:\n",
+      "\n",
+      "  bcrypt             conda-forge/linux-64::bcrypt-3.2.0-py39h3811e60_1\n",
+      "  dvc-ssh            conda-forge/linux-64::dvc-ssh-2.3.0-py39hf3d152e_0\n",
+      "  invoke             conda-forge/noarch::invoke-1.5.0-pyhd3deb0d_0\n",
+      "  paramiko           conda-forge/noarch::paramiko-2.7.2-pyh9f0ad1d_0\n",
+      "  pynacl             conda-forge/linux-64::pynacl-1.4.0-py39h3811e60_2\n",
+      "\n",
+      "The following packages will be UPDATED:\n",
+      "\n",
+      "  ca-certificates                      2020.12.5-ha878542_0 --> 2021.5.30-ha878542_0\n",
+      "  certifi                          2020.12.5-py39hf3d152e_1 --> 2021.5.30-py39hf3d152e_0\n",
+      "  dvc                                  2.1.0-py39hf3d152e_0 --> 2.3.0-py39hf3d152e_0\n",
+      "  fsspec                                 0.9.0-pyhd8ed1ab_2 --> 2021.5.0-pyhd8ed1ab_0\n",
+      "\n",
+      "\n",
+      "\n",
+      "Downloading and Extracting Packages\n",
+      "certifi-2021.5.30    | 141 KB    | ##################################### | 100% \n",
+      "fsspec-2021.5.0      | 77 KB     | ##################################### | 100% \n",
+      "dvc-2.3.0            | 542 KB    | ##################################### | 100% \n",
+      "invoke-1.5.0         | 137 KB    | ##################################### | 100% \n",
+      "paramiko-2.7.2       | 135 KB    | ##################################### | 100% \n",
+      "bcrypt-3.2.0         | 44 KB     | ##################################### | 100% \n",
+      "pynacl-1.4.0         | 1.3 MB    | ##################################### | 100% \n",
+      "dvc-ssh-2.3.0        | 9 KB      | ##################################### | 100% \n",
+      "ca-certificates-2021 | 136 KB    | ##################################### | 100% \n",
+      "Preparing transaction: done\n",
+      "Verifying transaction: done\n",
+      "Executing transaction: done\n",
+      "\n",
+      "Note: you may need to restart the kernel to use updated packages.\n"
+     ]
+    }
+   ],
+   "source": [
+    "conda install -c conda-forge dvc-ssh"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "e9a04876",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Setting 'ium_ssh_remote' as a default remote.\n",
+      "\u001b[0m"
+     ]
+    }
+   ],
+   "source": [
+    "!dvc remote add -f -d ium_ssh_remote ssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl/ium-sftp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "e3f27bbb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "my_local_remote\t/dvcstore\n",
+      "ium_ssh_remote\tssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl/ium-sftp\n",
+      "\u001b[0m"
+     ]
+    }
+   ],
+   "source": [
+    "!dvc remote list"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "5b2fa175",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[0m"
+     ]
+    }
+   ],
+   "source": [
+    "!dvc remote modify --local ium_ssh_remote password [hasło takie jak do serwera MLflow (patrz MSTeams)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "ea6e16fa",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "slide"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "  0% Uploading|                                      |0/1 [00:00<?,     ?file/s]\n",
+      "!\u001b[A\n",
+      "  0%|          |data/Iris.csv                  0.00/4.95k [00:00<?,        ?B/s]\u001b[A\n",
+      "1 file pushed                                                                   \u001b[A\n",
+      "\u001b[0m"
+     ]
+    }
+   ],
+   "source": [
+    "!dvc push"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1468c44c",
+   "metadata": {},
+   "source": [
+    "### Jenkins\n",
+    "\n",
+    "W Jenkins można użyć mechanizmu \"Credentials\", żeby w bezpieczny sposób przekazać hasło albo klucz prywatny.\n",
+    "\n",
+    "Takie dane dla użytkownika ium-sftp zostały stworzone na Jenkinsie:\n",
+    "\n",
+    " - typu ssh key: https://tzietkiewicz.vm.wmi.amu.edu.pl:8080/credentials/store/system/domain/_/credential/48ac7004-216e-4260-abba-1fe5db753e18/\n",
+    " - typu \"secret text\" - zawierający hasło użytkownika ium-shftp: https://tzietkiewicz.vm.wmi.amu.edu.pl:8080/credentials/store/system/domain/_/credential/ium-sftp-password/\n",
+    "\n",
+    "Opis używania \"Credentials\" w Jenkinsfile: https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#for-other-credential-types\n",
+    "\n",
+    "Klucza ssh można użyć tak: \n",
+    "\n",
+    "```Jenkinsfile\n",
+    "withCredentials(\n",
+    "    [sshUserPrivateKey(credentialsId: '48ac7004-216e-4260-abba-1fe5db753e18', keyFileVariable: 'IUM_SFTP_KEY', passphraseVariable: '', usernameVariable: '')]) {\n",
+    "                sh \"dvc init -f\"\n",
+    "                sh \"dvc remote add -d ium_ssh_remote ssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl/ium-sftp\"\n",
+    "                sh \"dvc remote modify --local ium_ssh_remote keyfile $IUM_SFTP_KEY\"\n",
+    "                sh \"dvc pull\"}\n",
+    "```\n",
+    "\n",
+    "Secret text tak:\n",
+    "\n",
+    "```Jenkinsfile\n",
+    "    withCredentials([string(credentialsId: 'ium-sftp-password', variable: 'IUM_SFTP_PASS')]) {\n",
+    "                sh \"dvc init -f\"\n",
+    "                sh \"dvc remote add -d ium_ssh_remote ssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl/ium-sftp\"\n",
+    "                sh \"dvc remote modify --local ium_ssh_remote password $IUM_SFTP_KEY\"\n",
+    "                sh \"dvc pull\"\n",
+    "    }\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "83778bf3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
diff --git a/IUM_12.Praca.ipynb b/IUM_12.Praca.ipynb
index ea4bbca..c5f5958 100644
--- a/IUM_12.Praca.ipynb
+++ b/IUM_12.Praca.ipynb
@@ -13,7 +13,7 @@
    "id": "69587a1e",
    "metadata": {},
    "source": [
-    "## Zadanie\n",
+    "## Zadanie [30pkt]\n",
     "\n",
     "1. Znajdź minimum 6, interesujących dla Ciebie, ogłoszeń o pracę\n",
     "  - na razie nie zważaj na wymagania\n",