From 7e8223b1f198534734ece544981cc55a5a55e31c Mon Sep 17 00:00:00 2001 From: Jakub Pokrywka Date: Wed, 1 Jun 2022 10:00:34 +0200 Subject: [PATCH] 10 --- cw/10_CRF.ipynb | 298 +++++++++--------------------------------------- 1 file changed, 56 insertions(+), 242 deletions(-) diff --git a/cw/10_CRF.ipynb b/cw/10_CRF.ipynb index 62f8d51..63c80c7 100644 --- a/cw/10_CRF.ipynb +++ b/cw/10_CRF.ipynb @@ -37,6 +37,16 @@ "https://www.aclweb.org/anthology/W03-0419.pdf" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "conda env export -n > environment.yml\n", + " \n", + " \n", + "conda env create -f path/to/environment.yml" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -56,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -86,7 +96,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": { "scrolled": false }, @@ -101,7 +111,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "60fb8337cb5b4ab28969b9e1d60a851c", + "model_id": "960f4cf0de594e48ad7a84740cf301a3", "version_major": 2, "version_minor": 0 }, @@ -119,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -134,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -143,7 +153,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -152,7 +162,7 @@ "21" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -163,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -173,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -183,7 +193,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -192,7 +202,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -201,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -210,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": { "scrolled": true }, @@ -221,7 +231,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -230,7 +240,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -239,7 +249,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -248,7 +258,7 @@ "tensor([ 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 3])" ] }, - "execution_count": 14, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -259,7 +269,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -307,7 +317,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -316,7 +326,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -336,7 +346,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -345,7 +355,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": { "scrolled": true }, @@ -356,7 +366,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -367,7 +377,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -389,7 +399,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -406,7 +416,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fc9748f7f63c47fea274592f4dba2c73", + "model_id": "caddabb06f894f529ba7143ce62b8c2e", "version_major": 2, "version_minor": 0 }, @@ -418,32 +428,11 @@ "output_type": "display_data" }, { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "> \u001b[0;32m/tmp/ipykernel_306568/4048919537.py\u001b[0m(12)\u001b[0;36m\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32m 10 \u001b[0;31m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0mcrf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0memissions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0m\u001b[0;32m 11 \u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpdb\u001b[0m\u001b[0;34m;\u001b[0m \u001b[0mpdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0m\u001b[0;32m---> 12 \u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0m\u001b[0;32m 13 \u001b[0;31m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0m\u001b[0;32m 14 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0m\n", - "ipdb> batch_tokens\n", - "tensor([ 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 3])\n", - "ipdb> tags.shape\n", - "torch.Size([11, 1])\n", - "ipdb> tags\n", - "tensor([[0],\n", - " [3],\n", - " [0],\n", - " [7],\n", - " [0],\n", - " [0],\n", - " [0],\n", - " [7],\n", - " [0],\n", - " [0],\n", - " [0]])\n" + "/home/kuba/anaconda3/envs/zajeciaei/lib/python3.10/site-packages/torchcrf/__init__.py:249: UserWarning: where received a uint8 condition tensor. This behavior is deprecated and will be removed in a future version of PyTorch. Use a boolean condition instead. (Triggered internally at /opt/conda/conda-bld/pytorch_1646755897462/work/aten/src/ATen/native/TensorCompare.cpp:333.)\n", + " score = torch.where(mask[i].unsqueeze(1), next_score, score)\n" ] } ], @@ -458,7 +447,7 @@ "\n", " optimizer.zero_grad()\n", " loss = -crf(emissions,tags)\n", - " import pdb; pdb.set_trace()\n", + " #import pdb; pdb.set_trace()\n", " loss.backward()\n", " optimizer.step()\n", " \n", @@ -469,213 +458,40 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['T_destination',\n", - " '__annotations__',\n", - " '__call__',\n", - " '__class__',\n", - " '__delattr__',\n", - " '__dict__',\n", - " '__dir__',\n", - " '__doc__',\n", - " '__eq__',\n", - " '__format__',\n", - " '__ge__',\n", - " '__getattr__',\n", - " '__getattribute__',\n", - " '__gt__',\n", - " '__hash__',\n", - " '__init__',\n", - " '__init_subclass__',\n", - " '__le__',\n", - " '__lt__',\n", - " '__module__',\n", - " '__ne__',\n", - " '__new__',\n", - " '__reduce__',\n", - " '__reduce_ex__',\n", - " '__repr__',\n", - " '__setattr__',\n", - " '__setstate__',\n", - " '__sizeof__',\n", - " '__str__',\n", - " '__subclasshook__',\n", - " '__weakref__',\n", - " '_apply',\n", - " '_backward_hooks',\n", - " '_buffers',\n", - " '_call_impl',\n", - " '_compute_normalizer',\n", - " '_compute_score',\n", - " '_forward_hooks',\n", - " '_forward_pre_hooks',\n", - " '_get_backward_hooks',\n", - " '_get_name',\n", - " '_is_full_backward_hook',\n", - " '_load_from_state_dict',\n", - " '_load_state_dict_pre_hooks',\n", - " '_maybe_warn_non_full_backward_hook',\n", - " '_modules',\n", - " '_named_members',\n", - " '_non_persistent_buffers_set',\n", - " '_parameters',\n", - " '_register_load_state_dict_pre_hook',\n", - " '_register_state_dict_hook',\n", - " '_replicate_for_data_parallel',\n", - " '_save_to_state_dict',\n", - " '_slow_forward',\n", - " '_state_dict_hooks',\n", - " '_validate',\n", - " '_version',\n", - " '_viterbi_decode',\n", - " 'add_module',\n", - " 'apply',\n", - " 'batch_first',\n", - " 'bfloat16',\n", - " 'buffers',\n", - " 'children',\n", - " 'cpu',\n", - " 'cuda',\n", - " 'decode',\n", - " 'double',\n", - " 'dump_patches',\n", - " 'end_transitions',\n", - " 'eval',\n", - " 'extra_repr',\n", - " 'float',\n", - " 'forward',\n", - " 'get_buffer',\n", - " 'get_extra_state',\n", - " 'get_parameter',\n", - " 'get_submodule',\n", - " 'half',\n", - " 'load_state_dict',\n", - " 'modules',\n", - " 'named_buffers',\n", - " 'named_children',\n", - " 'named_modules',\n", - " 'named_parameters',\n", - " 'num_tags',\n", - " 'parameters',\n", - " 'register_backward_hook',\n", - " 'register_buffer',\n", - " 'register_forward_hook',\n", - " 'register_forward_pre_hook',\n", - " 'register_full_backward_hook',\n", - " 'register_module',\n", - " 'register_parameter',\n", - " 'requires_grad_',\n", - " 'reset_parameters',\n", - " 'set_extra_state',\n", - " 'share_memory',\n", - " 'start_transitions',\n", - " 'state_dict',\n", - " 'to',\n", - " 'to_empty',\n", - " 'train',\n", - " 'training',\n", - " 'transitions',\n", - " 'type',\n", - " 'xpu',\n", - " 'zero_grad']" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dir(crf)\n" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Parameter containing:\n", - "tensor([[ 0.1427, 0.0082, -0.0852, -0.0714, -0.0514, 0.0753, 0.0389, 0.0018,\n", - " -0.0806],\n", - " [-0.0809, -0.0508, 0.0520, -0.0619, 0.0181, -0.0729, -0.1430, -0.1055,\n", - " 0.0384],\n", - " [-0.0011, -0.1476, 0.0425, -0.0081, -0.1181, -0.0098, -0.0567, 0.0311,\n", - " -0.0696],\n", - " [-0.0443, -0.0741, 0.0463, -0.0967, -0.0403, -0.0243, 0.0098, -0.0063,\n", - " -0.0811],\n", - " [ 0.0632, -0.1175, -0.0992, 0.0198, 0.0310, -0.0059, 0.0191, -0.1303,\n", - " -0.1423],\n", - " [ 0.0029, 0.0296, 0.0152, -0.0418, -0.1068, -0.0920, -0.0380, 0.0461,\n", - " 0.0167],\n", - " [-0.1167, -0.0559, -0.0428, -0.0115, -0.1006, -0.1511, 0.0035, -0.0273,\n", - " -0.1201],\n", - " [-0.0378, 0.0481, -0.1474, -0.0154, 0.0347, -0.0392, -0.0755, -0.1227,\n", - " 0.0448],\n", - " [-0.0383, -0.0402, 0.0054, 0.0145, -0.1353, -0.0460, 0.0257, -0.0322,\n", - " 0.0023]], requires_grad=True)" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "crf.transitions" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Parameter containing:\n", - " tensor([-0.0432, -0.1150, -0.1045, -0.0779, -0.0858, 0.0287, -0.1437, -0.1446,\n", - " 0.0335], requires_grad=True),\n", - " Parameter containing:\n", - " tensor([ 0.0838, -0.0097, -0.1136, 0.0010, -0.1177, 0.0225, 0.0292, -0.0837,\n", - " -0.1063], requires_grad=True),\n", - " Parameter containing:\n", - " tensor([[ 0.1427, 0.0082, -0.0852, -0.0714, -0.0514, 0.0753, 0.0389, 0.0018,\n", - " -0.0806],\n", - " [-0.0809, -0.0508, 0.0520, -0.0619, 0.0181, -0.0729, -0.1430, -0.1055,\n", - " 0.0384],\n", - " [-0.0011, -0.1476, 0.0425, -0.0081, -0.1181, -0.0098, -0.0567, 0.0311,\n", - " -0.0696],\n", - " [-0.0443, -0.0741, 0.0463, -0.0967, -0.0403, -0.0243, 0.0098, -0.0063,\n", - " -0.0811],\n", - " [ 0.0632, -0.1175, -0.0992, 0.0198, 0.0310, -0.0059, 0.0191, -0.1303,\n", - " -0.1423],\n", - " [ 0.0029, 0.0296, 0.0152, -0.0418, -0.1068, -0.0920, -0.0380, 0.0461,\n", - " 0.0167],\n", - " [-0.1167, -0.0559, -0.0428, -0.0115, -0.1006, -0.1511, 0.0035, -0.0273,\n", - " -0.1201],\n", - " [-0.0378, 0.0481, -0.1474, -0.0154, 0.0347, -0.0392, -0.0755, -0.1227,\n", - " 0.0448],\n", - " [-0.0383, -0.0402, 0.0054, 0.0145, -0.1353, -0.0460, 0.0257, -0.0322,\n", - " 0.0023]], requires_grad=True)]" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "list(crf.parameters())" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_model(validation_tokens_ids, validation_labels)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -688,9 +504,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "eval_model(validation_tokens_ids, validation_labels)" - ] + "source": [] }, { "cell_type": "code",