diff --git a/train/RNN.ipynb b/train/RNN.ipynb index fad0feb..19c4160 100644 --- a/train/RNN.ipynb +++ b/train/RNN.ipynb @@ -28,7 +28,6 @@ "from torchtext.vocab import vocab\n", "from tqdm.notebook import tqdm\n", "import pandas as pd\n", - "import nltk\n", "from nltk.tokenize import word_tokenize\n", "import string" ] @@ -51,9 +50,7 @@ "outputs": [], "source": [ "X_test = pd.read_csv('../dev-0/in.tsv', sep='\\t', header=None)\n", - "Y_test = pd.read_csv('../dev-0/expected.tsv', sep='\\t', header=None)\n", - "\n", - "X_val = pd.read_csv('../test-A/in.tsv', sep='\\t', header=None)" + "Y_test = pd.read_csv('../dev-0/expected.tsv', sep='\\t', header=None)" ] }, { @@ -68,7 +65,8 @@ "text": [ "German July car registrations up 14.2 pct yr / yr . FRANKFURT 1996-08-22 German first-time registrations of motor vehicles jumped 14.2 percent in July this year from the year-earlier period , the Federal office for motor vehicles said on Thursday . The office said 356,725 new cars were registered in July 1996 -- 304,850 passenger cars and 15,613 trucks . The figures represent a 13.6 percent increase for passenger cars and a 2.2 percent decline for trucks from July 1995 . Motor-bike registration rose 32.7 percent in the period . The growth was partly due to an increased number of Germans buying German cars abroad , while manufacturers said that domestic demand was weak , the federal office said . Almost all German car manufacturers posted gains in registration numbers in the period . Volkswagen AG won 77,719 registrations , slightly more than a quarter of the total . Opel AG together with General Motors came in second place with 49,269 registrations , 16.4 percent of the overall figure . Third was Ford with 35,563 registrations , or 11.7 percent . Only Seat and Porsche had fewer registrations in July 1996 compared to last year 's July . Seat posted 3,420 registrations compared with 5522 registrations in July a year earlier . Porsche 's registrations fell to 554 from 643 . \n", "BASKETBALL - INTERNATIONAL TOURNAMENT RESULT . BELGRADE 1996-08-30 Result in an international basketball tournament on Friday : Red Star ( Yugoslavia ) beat Dinamo ( Russia ) 92-90 ( halftime 47-47 ) \n", - "SOCCER - ASIAN CUP GROUP C RESULTS . AL-AIN , United Arab Emirates 1996-12-06 Results of Asian Cup group C matches played on Friday : Japan 2 Syria 1 ( halftime 0-1 ) Scorers : Japan - Hassan Abbas 84 own goal , Takuya Takagi 88 . Syria - Nader Jokhadar 7 Attendance : 10,000 . China 0 Uzbekistan 2 ( halftime 0-0 ) Scorers : Shkvyrin Igor 78 , Shatskikh Oleg 90 Attendence : 3,000 Standings ( tabulate under played , won , drawn , lost , goals for , goals against , points ) : Uzbekistan 1 1 0 0 2 0 3 Japan 1 1 0 0 2 1 3 Syria 1 0 0 1 1 2 0 China 1 0 0 1 0 2 0 \n" + "B-MISC O O O O O O O O O O O B-LOC O O B-MISC O O O O O O O O O O O O O O O O O O B-ORG I-ORG I-ORG I-ORG I-ORG O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-MISC O B-MISC O O O O O O O O O O O O O O O O O O O O B-MISC O O O O O O O O O O O O B-ORG I-ORG O O O O O O O O O O O O O O B-ORG I-ORG O O B-ORG I-ORG O O O O O O O O O O O O O O O O O O B-ORG O O O O O O O O O O B-ORG O B-ORG O O O O O O O O O O O O O O B-ORG O O O O O O O O O O O O O O B-ORG O O O O O O O O O\n", + "O O O O O O O B-LOC O O O O O O O O O O O O O B-ORG I-ORG O B-LOC O O B-ORG O B-LOC O O O O O O O O\n" ] } ], @@ -79,11 +77,10 @@ "X_test = X_test[X_test.columns[0]].replace(\"\",\"\")\n", "Y_test = Y_test[Y_test.columns[0]]\n", "\n", - "X_val = X_val[X_val.columns[0]].replace(\"\",\"\")\n", - "\n", "print(X_train[4])\n", "print(X_test[4])\n", - "print(X_val[4])" + "print(Y_train[4])\n", + "print(Y_test[4])" ] }, { @@ -97,42 +94,33 @@ "output_type": "stream", "text": [ " ['german', 'july', 'car', 'registrations', 'up', '14.2', 'pct', 'yr', '/', 'yr', '.', '', 'frankfurt', '1996-08-22', '', 'german', 'first-time', 'registrations', 'of', 'motor', 'vehicles', 'jumped', '14.2', 'percent', 'in', 'july', 'this', 'year', 'from', 'the', 'year-earlier', 'period', ',', 'the', 'federal', 'office', 'for', 'motor', 'vehicles', 'said', 'on', 'thursday', '.', '', 'the', 'office', 'said', '356,725', 'new', 'cars', 'were', 'registered', 'in', 'july', '1996', '--', '304,850', 'passenger', 'cars', 'and', '15,613', 'trucks', '.', '', 'the', 'figures', 'represent', 'a', '13.6', 'percent', 'increase', 'for', 'passenger', 'cars', 'and', 'a', '2.2', 'percent', 'decline', 'for', 'trucks', 'from', 'july', '1995', '.', '', 'motor-bike', 'registration', 'rose', '32.7', 'percent', 'in', 'the', 'period', '.', '', 'the', 'growth', 'was', 'partly', 'due', 'to', 'an', 'increased', 'number', 'of', 'germans', 'buying', 'german', 'cars', 'abroad', ',', 'while', 'manufacturers', 'said', 'that', 'domestic', 'demand', 'was', 'weak', ',', 'the', 'federal', 'office', 'said', '.', '', 'almost', 'all', 'german', 'car', 'manufacturers', 'posted', 'gains', 'in', 'registration', 'numbers', 'in', 'the', 'period', '.', '', 'volkswagen', 'ag', 'won', '77,719', 'registrations', ',', 'slightly', 'more', 'than', 'a', 'quarter', 'of', 'the', 'total', '.', '', 'opel', 'ag', 'together', 'with', 'general', 'motors', 'came', 'in', 'second', 'place', 'with', '49,269', 'registrations', ',', '16.4', 'percent', 'of', 'the', 'overall', 'figure', '.', '', 'third', 'was', 'ford', 'with', '35,563', 'registrations', ',', 'or', '11.7', 'percent', '.', '', 'only', 'seat', 'and', 'porsche', 'had', 'fewer', 'registrations', 'in', 'july', '1996', 'compared', 'to', 'last', 'year', \"'s\", 'july', '.', '', 'seat', 'posted', '3,420', 'registrations', 'compared', 'with', '5522', 'registrations', 'in', 'july', 'a', 'year', 'earlier', '.', '', 'porsche', \"'s\", 'registrations', 'fell', 'to', '554', 'from', '643', '.', '']\n", - " ['basketball', '-', 'international', 'tournament', 'result', '.', '', 'belgrade', '1996-08-30', '', 'result', 'in', 'an', 'international', '', 'basketball', 'tournament', 'on', 'friday', ':', '', 'red', 'star', '(', 'yugoslavia', ')', 'beat', 'dinamo', '(', 'russia', ')', '92-90', '(', 'halftime', '', '47-47', ')', '']\n", - " ['soccer', '-', 'asian', 'cup', 'group', 'c', 'results', '.', '', 'al-ain', ',', 'united', 'arab', 'emirates', '1996-12-06', '', 'results', 'of', 'asian', 'cup', 'group', 'c', 'matches', 'played', 'on', 'friday', ':', '', 'japan', '2', 'syria', '1', '(', 'halftime', '0-1', ')', '', 'scorers', ':', '', 'japan', '-', 'hassan', 'abbas', '84', 'own', 'goal', ',', 'takuya', 'takagi', '88', '.', '', 'syria', '-', 'nader', 'jokhadar', '7', '', 'attendance', ':', '10,000', '.', '', 'china', '0', 'uzbekistan', '2', '(', 'halftime', '0-0', ')', '', 'scorers', ':', 'shkvyrin', 'igor', '78', ',', 'shatskikh', 'oleg', '90', '', 'attendence', ':', '3,000', '', 'standings', '(', 'tabulate', 'under', 'played', ',', 'won', ',', 'drawn', ',', 'lost', ',', 'goals', '', 'for', ',', 'goals', 'against', ',', 'points', ')', ':', '', 'uzbekistan', '1', '1', '0', '0', '2', '0', '3', '', 'japan', '1', '1', '0', '0', '2', '1', '3', '', 'syria', '1', '0', '0', '1', '1', '2', '0', '', 'china', '1', '0', '0', '1', '0', '2', '0', '']\n" + " ['basketball', '-', 'international', 'tournament', 'result', '.', '', 'belgrade', '1996-08-30', '', 'result', 'in', 'an', 'international', '', 'basketball', 'tournament', 'on', 'friday', ':', '', 'red', 'star', '(', 'yugoslavia', ')', 'beat', 'dinamo', '(', 'russia', ')', '92-90', '(', 'halftime', '', '47-47', ')', '']\n" ] } ], "source": [ - "def preprocess(text):\n", - " text = text.lower()\n", - " return text\n", - "\n", - "X_train = [preprocess(text).split() for text in X_train]\n", + "X_train = [text.lower().split() for text in X_train]\n", "print( type(X_train), X_train[4])\n", - "X_test = [preprocess(text).split() for text in X_test]\n", - "print( type(X_test), X_test[4])\n", - "X_val = [preprocess(text).split() for text in X_val]\n", - "print( type(X_val), X_val[4])" + "X_test = [text.lower().split() for text in X_test]\n", + "print( type(X_test), X_test[4])" ] }, { "cell_type": "code", "execution_count": 6, - "id": "c24df99f-1792-4691-92c0-7e5e596237c3", + "id": "cd8e7040-d169-466b-bb7a-74e1c1a7fec0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['german', 'july', 'car', 'registrations', 'up', '14.2', 'pct', 'yr', '/', 'yr', '.', '', 'frankfurt', '1996-08-22', '', 'german', 'first-time', 'registrations', 'of', 'motor', 'vehicles', 'jumped', '14.2', 'percent', 'in', 'july', 'this', 'year', 'from', 'the', 'year-earlier', 'period', ',', 'the', 'federal', 'office', 'for', 'motor', 'vehicles', 'said', 'on', 'thursday', '.', '', 'the', 'office', 'said', '356,725', 'new', 'cars', 'were', 'registered', 'in', 'july', '1996', '--', '304,850', 'passenger', 'cars', 'and', '15,613', 'trucks', '.', '', 'the', 'figures', 'represent', 'a', '13.6', 'percent', 'increase', 'for', 'passenger', 'cars', 'and', 'a', '2.2', 'percent', 'decline', 'for', 'trucks', 'from', 'july', '1995', '.', '', 'motor-bike', 'registration', 'rose', '32.7', 'percent', 'in', 'the', 'period', '.', '', 'the', 'growth', 'was', 'partly', 'due', 'to', 'an', 'increased', 'number', 'of', 'germans', 'buying', 'german', 'cars', 'abroad', ',', 'while', 'manufacturers', 'said', 'that', 'domestic', 'demand', 'was', 'weak', ',', 'the', 'federal', 'office', 'said', '.', '', 'almost', 'all', 'german', 'car', 'manufacturers', 'posted', 'gains', 'in', 'registration', 'numbers', 'in', 'the', 'period', '.', '', 'volkswagen', 'ag', 'won', '77,719', 'registrations', ',', 'slightly', 'more', 'than', 'a', 'quarter', 'of', 'the', 'total', '.', '', 'opel', 'ag', 'together', 'with', 'general', 'motors', 'came', 'in', 'second', 'place', 'with', '49,269', 'registrations', ',', '16.4', 'percent', 'of', 'the', 'overall', 'figure', '.', '', 'third', 'was', 'ford', 'with', '35,563', 'registrations', ',', 'or', '11.7', 'percent', '.', '', 'only', 'seat', 'and', 'porsche', 'had', 'fewer', 'registrations', 'in', 'july', '1996', 'compared', 'to', 'last', 'year', \"'s\", 'july', '.', '', 'seat', 'posted', '3,420', 'registrations', 'compared', 'with', '5522', 'registrations', 'in', 'july', 'a', 'year', 'earlier', '.', '', 'porsche', \"'s\", 'registrations', 'fell', 'to', '554', 'from', '643', '.', ''] B-MISC O O O O O O O O O O O B-LOC O O B-MISC O O O O O O O O O O O O O O O O O O B-ORG I-ORG I-ORG I-ORG I-ORG O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-MISC O B-MISC O O O O O O O O O O O O O O O O O O O O B-MISC O O O O O O O O O O O O B-ORG I-ORG O O O O O O O O O O O O O O B-ORG I-ORG O O B-ORG I-ORG O O O O O O O O O O O O O O O O O O B-ORG O O O O O O O O O O B-ORG O B-ORG O O O O O O O O O O O O O O B-ORG O O O O O O O O O O O O O O B-ORG O O O O O O O O O ['soccer', '-', 'asian', 'cup', 'group', 'c', 'results', '.', '', 'al-ain', ',', 'united', 'arab', 'emirates', '1996-12-06', '', 'results', 'of', 'asian', 'cup', 'group', 'c', 'matches', 'played', 'on', 'friday', ':', '', 'japan', '2', 'syria', '1', '(', 'halftime', '0-1', ')', '', 'scorers', ':', '', 'japan', '-', 'hassan', 'abbas', '84', 'own', 'goal', ',', 'takuya', 'takagi', '88', '.', '', 'syria', '-', 'nader', 'jokhadar', '7', '', 'attendance', ':', '10,000', '.', '', 'china', '0', 'uzbekistan', '2', '(', 'halftime', '0-0', ')', '', 'scorers', ':', 'shkvyrin', 'igor', '78', ',', 'shatskikh', 'oleg', '90', '', 'attendence', ':', '3,000', '', 'standings', '(', 'tabulate', 'under', 'played', ',', 'won', ',', 'drawn', ',', 'lost', ',', 'goals', '', 'for', ',', 'goals', 'against', ',', 'points', ')', ':', '', 'uzbekistan', '1', '1', '0', '0', '2', '0', '3', '', 'japan', '1', '1', '0', '0', '2', '1', '3', '', 'syria', '1', '0', '0', '1', '1', '2', '0', '', 'china', '1', '0', '0', '1', '0', '2', '0', '']\n", - "235 562\n" + "489 1265\n" ] } ], "source": [ - "print(X_train[4], Y_train[4], X_val[4])\n", - "print(len(X_train[4]), len(Y_train[4]))" + "print(len(X_train[0]), len(Y_train[0]))" ] }, { @@ -179,12 +167,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "['', '', '', '', 'eu', 'rejects', 'german', 'call', 'to', 'boycott', 'british']\n" + "['', '', '', '', 'eu', 'rejects', 'german', 'call', 'to', 'boycott']\n" ] } ], "source": [ - "print(itos[:11])" + "print(itos[:10])" ] }, { @@ -299,22 +287,7 @@ "tensor([ 2, 9637, 640, 419, 1908, 1850, 12, 13, 2439, 19358,\n", " 13, 1850, 233, 159, 419, 13, 9637, 1908, 22, 1098,\n", " 380, 13, 1103, 2438, 132, 2440, 134, 1762, 11167, 132,\n", - " 1160, 134, 0, 132, 1767, 13, 0, 134, 13, 3])\n", - "tensor([ 2, 1759, 640, 5613, 1770, 391, 2103, 1301, 12, 13,\n", - " 0, 73, 820, 1077, 1078, 0, 13, 1301, 163, 5613,\n", - " 1770, 391, 2103, 2010, 2489, 22, 1098, 380, 13, 1677,\n", - " 657, 667, 1316, 132, 1767, 6515, 134, 13, 1775, 380,\n", - " 13, 1677, 640, 875, 18629, 5763, 961, 426, 73, 0,\n", - " 0, 9355, 12, 13, 667, 640, 0, 0, 1929, 13,\n", - " 1786, 380, 4031, 12, 13, 345, 1577, 0, 657, 132,\n", - " 1767, 2299, 134, 13, 1775, 380, 0, 0, 9302, 73,\n", - " 0, 0, 4497, 13, 17180, 380, 2354, 13, 2853, 132,\n", - " 2854, 124, 2489, 73, 491, 73, 3958, 73, 2855, 73,\n", - " 2357, 13, 72, 73, 2357, 746, 73, 1469, 134, 380,\n", - " 13, 0, 1316, 1316, 1577, 1577, 657, 1577, 1945, 13,\n", - " 1677, 1316, 1316, 1577, 1577, 657, 1316, 1945, 13, 667,\n", - " 1316, 1577, 1577, 1316, 1316, 657, 1577, 13, 345, 1316,\n", - " 1577, 1577, 1316, 1577, 657, 1577, 13, 3])\n" + " 1160, 134, 0, 132, 1767, 13, 0, 134, 13, 3])\n" ] } ], @@ -322,9 +295,7 @@ "train_tokens_ids = data_process(X_train)\n", "print(train_tokens_ids[4])\n", "validation_tokens_ids = data_process(X_test)\n", - "print(validation_tokens_ids[4])\n", - "val_tokens_ids = data_process(X_val)\n", - "print(val_tokens_ids[4])" + "print(validation_tokens_ids[4])" ] }, { @@ -356,7 +327,7 @@ "output_type": "stream", "text": [ "9\n", - "{'B-LOC': 0, 'B-MISC': 1, 'I-PER': 2, 'B-PER': 3, 'I-MISC': 4, 'I-ORG': 5, 'B-ORG': 6, 'I-LOC': 7, 'O': 8}\n" + "{'I-PER': 0, 'B-PER': 1, 'O': 2, 'I-LOC': 3, 'I-MISC': 4, 'B-ORG': 5, 'B-LOC': 6, 'B-MISC': 7, 'I-ORG': 8}\n" ] } ], @@ -422,7 +393,7 @@ "output_type": "stream", "text": [ "O O B-ORG O O O O O O O O O B-LOC O O B-MISC I-MISC O B-PER I-PER O O O O O O O B-ORG O B-ORG O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-ORG O B-ORG O B-ORG O O O O O O B-ORG O O O O O O O O O O B-ORG O O O O B-ORG O O O O O O O O B-LOC I-LOC O B-ORG O O O O O O O O O O O O O O B-LOC O B-PER I-PER O O O O O O O O O O B-ORG O O O O O O O O O B-PER O O O O O O O O O O B-ORG O O O O O O O O O O O B-PER I-PER O B-PER I-PER O O O O O O O O O B-ORG O B-LOC O O B-PER O O O O B-LOC O O O O O O O O O O O O O O O O B-ORG O O O O O O O O O O O O O O O B-ORG O O O O O O O O O B-PER O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-LOC O B-ORG O B-PER I-PER O O O O O B-LOC O O O O O O O O O O O O O O O O O O O O O O O O O B-ORG O O O O O O O B-LOC O B-PER I-PER O O O O B-ORG O O O O O O O O O O O O B-ORG O O O O O O O O O O O O O B-ORG O O O O O O O O O O O O O O O O O O O O B-MISC B-PER I-PER O O O O O B-PER I-PER O O O O B-PER I-PER O O O O B-ORG O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-ORG O O O O O O B-ORG O O O O O O O O O O O O O B-PER I-PER O B-MISC O O B-PER I-PER O O O O O O O O B-ORG O O O O O O O\n", - "[8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 1, 4, 8, 3, 2, 8, 8, 8, 8, 8, 8, 8, 6, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 6, 8, 6, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 0, 7, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 3, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 2, 8, 3, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 0, 8, 8, 3, 8, 8, 8, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 6, 8, 3, 2, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 0, 8, 3, 2, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 1, 3, 2, 8, 8, 8, 8, 8, 3, 2, 8, 8, 8, 8, 3, 2, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 2, 8, 1, 8, 8, 3, 2, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8]\n" + "[2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 6, 2, 2, 7, 4, 2, 1, 0, 2, 2, 2, 2, 2, 2, 2, 5, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 5, 2, 5, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 6, 3, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 6, 2, 1, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 0, 2, 1, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 6, 2, 2, 1, 2, 2, 2, 2, 6, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 6, 2, 5, 2, 1, 0, 2, 2, 2, 2, 2, 6, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 6, 2, 1, 0, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 7, 1, 0, 2, 2, 2, 2, 2, 1, 0, 2, 2, 2, 2, 1, 0, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 0, 2, 7, 2, 2, 1, 0, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2]\n" ] } ], @@ -442,47 +413,47 @@ "name": "stdout", "output_type": "stream", "text": [ - "tensor([0, 6, 8, 1, 8, 8, 8, 1, 8, 8, 8, 3, 2, 8, 0, 8, 8, 8, 6, 5, 8, 8, 8, 8,\n", - " 8, 8, 1, 8, 8, 8, 8, 8, 1, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 0, 8, 8, 8, 8, 6, 5, 8, 8, 8, 3, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 6, 8, 8, 8, 3, 2, 2, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 5, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 6, 8, 8, 3, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 8, 1, 8, 8, 8, 8, 0, 8, 0, 8,\n", - " 8, 8, 8, 8, 8, 8, 1, 4, 4, 8, 1, 8, 8, 8, 8, 8, 8, 8, 8, 3, 8, 8, 8, 8,\n", - " 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 1, 8, 8, 3, 2, 2, 8, 8, 8, 3, 8, 8, 6, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 0, 8, 3, 8, 8, 8, 8,\n", - " 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 1, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 1, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 1, 8, 8, 8, 8, 8, 8, 1, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8, 6, 5, 5, 5,\n", - " 5, 8, 6, 8, 8, 3, 2, 2, 8, 8, 6, 5, 8, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 1, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 0, 8, 8, 8, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 1,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0])\n", - "tensor([0, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 1, 4, 8, 3, 2, 8, 8, 8,\n", - " 8, 8, 8, 8, 6, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 6, 8, 6, 8, 8,\n", - " 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 6, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 0, 7, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0,\n", - " 8, 3, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 2,\n", - " 8, 3, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 0, 8, 8, 3, 8, 8, 8, 8, 0, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8,\n", - " 6, 8, 3, 2, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 0, 8, 3, 2, 8,\n", - " 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 1, 3, 2, 8, 8, 8, 8, 8, 3, 2, 8, 8, 8, 8, 3, 2, 8, 8, 8, 8, 6,\n", - " 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n", - " 8, 3, 2, 8, 1, 8, 8, 3, 2, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8,\n", - " 8, 0])\n" + "tensor([0, 5, 2, 7, 2, 2, 2, 7, 2, 2, 2, 1, 0, 2, 6, 2, 2, 2, 5, 8, 2, 2, 2, 2,\n", + " 2, 2, 7, 2, 2, 2, 2, 2, 7, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 6, 2, 2, 2, 2, 5, 8, 2, 2, 2, 1, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 6,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 5, 2, 2, 2, 1, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 8, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 5, 2, 2, 1, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 7, 2, 2, 2, 2, 6, 2, 6, 2,\n", + " 2, 2, 2, 2, 2, 2, 7, 4, 4, 2, 7, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2,\n", + " 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 7, 2, 2, 1, 0, 0, 2, 2, 2, 1, 2, 2, 5, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 6, 2, 6, 2, 1, 2, 2, 2, 2,\n", + " 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 7, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 7, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 7, 2, 2, 2, 2, 2, 2, 7, 2, 2, 2, 2, 2, 6, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 6, 2, 2, 2, 2, 5, 8, 8, 8,\n", + " 8, 2, 5, 2, 2, 1, 0, 0, 2, 2, 5, 8, 2, 2, 6, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 7, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 6, 2, 2, 2, 2, 6, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 7,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0])\n", + "tensor([0, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 6, 2, 2, 7, 4, 2, 1, 0, 2, 2, 2,\n", + " 2, 2, 2, 2, 5, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 5, 2, 5, 2, 2,\n", + " 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 5, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 6, 3, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 6,\n", + " 2, 1, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 0,\n", + " 2, 1, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 6, 2, 2, 1, 2, 2, 2, 2, 6, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 6, 2,\n", + " 5, 2, 1, 0, 2, 2, 2, 2, 2, 6, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 6, 2, 1, 0, 2,\n", + " 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 7, 1, 0, 2, 2, 2, 2, 2, 1, 0, 2, 2, 2, 2, 1, 0, 2, 2, 2, 2, 5,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 1, 0, 2, 7, 2, 2, 1, 0, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2,\n", + " 2, 0])\n" ] } ], @@ -590,7 +561,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "['german', 'july', 'car', 'registrations', 'up', '14.2', 'pct', 'yr', '/', 'yr', '.', '', 'frankfurt', '1996-08-22', '', 'german', 'first-time', 'registrations', 'of', 'motor', 'vehicles', 'jumped', '14.2', 'percent', 'in', 'july', 'this', 'year', 'from', 'the', 'year-earlier', 'period', ',', 'the', 'federal', 'office', 'for', 'motor', 'vehicles', 'said', 'on', 'thursday', '.', '', 'the', 'office', 'said', '356,725', 'new', 'cars', 'were', 'registered', 'in', 'july', '1996', '--', '304,850', 'passenger', 'cars', 'and', '15,613', 'trucks', '.', '', 'the', 'figures', 'represent', 'a', '13.6', 'percent', 'increase', 'for', 'passenger', 'cars', 'and', 'a', '2.2', 'percent', 'decline', 'for', 'trucks', 'from', 'july', '1995', '.', '', 'motor-bike', 'registration', 'rose', '32.7', 'percent', 'in', 'the', 'period', '.', '', 'the', 'growth', 'was', 'partly', 'due', 'to', 'an', 'increased', 'number', 'of', 'germans', 'buying', 'german', 'cars', 'abroad', ',', 'while', 'manufacturers', 'said', 'that', 'domestic', 'demand', 'was', 'weak', ',', 'the', 'federal', 'office', 'said', '.', '', 'almost', 'all', 'german', 'car', 'manufacturers', 'posted', 'gains', 'in', 'registration', 'numbers', 'in', 'the', 'period', '.', '', 'volkswagen', 'ag', 'won', '77,719', 'registrations', ',', 'slightly', 'more', 'than', 'a', 'quarter', 'of', 'the', 'total', '.', '', 'opel', 'ag', 'together', 'with', 'general', 'motors', 'came', 'in', 'second', 'place', 'with', '49,269', 'registrations', ',', '16.4', 'percent', 'of', 'the', 'overall', 'figure', '.', '', 'third', 'was', 'ford', 'with', '35,563', 'registrations', ',', 'or', '11.7', 'percent', '.', '', 'only', 'seat', 'and', 'porsche', 'had', 'fewer', 'registrations', 'in', 'july', '1996', 'compared', 'to', 'last', 'year', \"'s\", 'july', '.', '', 'seat', 'posted', '3,420', 'registrations', 'compared', 'with', '5522', 'registrations', 'in', 'july', 'a', 'year', 'earlier', '.', '', 'porsche', \"'s\", 'registrations', 'fell', 'to', '554', 'from', '643', '.', ''] [1, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 1, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 5, 5, 5, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 1, 8, 1, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 1, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 5, 8, 8, 6, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8]\n", + "['german', 'july', 'car', 'registrations', 'up', '14.2', 'pct', 'yr', '/', 'yr', '.', '', 'frankfurt', '1996-08-22', '', 'german', 'first-time', 'registrations', 'of', 'motor', 'vehicles', 'jumped', '14.2', 'percent', 'in', 'july', 'this', 'year', 'from', 'the', 'year-earlier', 'period', ',', 'the', 'federal', 'office', 'for', 'motor', 'vehicles', 'said', 'on', 'thursday', '.', '', 'the', 'office', 'said', '356,725', 'new', 'cars', 'were', 'registered', 'in', 'july', '1996', '--', '304,850', 'passenger', 'cars', 'and', '15,613', 'trucks', '.', '', 'the', 'figures', 'represent', 'a', '13.6', 'percent', 'increase', 'for', 'passenger', 'cars', 'and', 'a', '2.2', 'percent', 'decline', 'for', 'trucks', 'from', 'july', '1995', '.', '', 'motor-bike', 'registration', 'rose', '32.7', 'percent', 'in', 'the', 'period', '.', '', 'the', 'growth', 'was', 'partly', 'due', 'to', 'an', 'increased', 'number', 'of', 'germans', 'buying', 'german', 'cars', 'abroad', ',', 'while', 'manufacturers', 'said', 'that', 'domestic', 'demand', 'was', 'weak', ',', 'the', 'federal', 'office', 'said', '.', '', 'almost', 'all', 'german', 'car', 'manufacturers', 'posted', 'gains', 'in', 'registration', 'numbers', 'in', 'the', 'period', '.', '', 'volkswagen', 'ag', 'won', '77,719', 'registrations', ',', 'slightly', 'more', 'than', 'a', 'quarter', 'of', 'the', 'total', '.', '', 'opel', 'ag', 'together', 'with', 'general', 'motors', 'came', 'in', 'second', 'place', 'with', '49,269', 'registrations', ',', '16.4', 'percent', 'of', 'the', 'overall', 'figure', '.', '', 'third', 'was', 'ford', 'with', '35,563', 'registrations', ',', 'or', '11.7', 'percent', '.', '', 'only', 'seat', 'and', 'porsche', 'had', 'fewer', 'registrations', 'in', 'july', '1996', 'compared', 'to', 'last', 'year', \"'s\", 'july', '.', '', 'seat', 'posted', '3,420', 'registrations', 'compared', 'with', '5522', 'registrations', 'in', 'july', 'a', 'year', 'earlier', '.', '', 'porsche', \"'s\", 'registrations', 'fell', 'to', '554', 'from', '643', '.', ''] [7, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 6, 2, 2, 7, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 8, 8, 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 7, 2, 7, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 7, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 8, 2, 2, 5, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 2, 2, 2, 2, 2, 2, 2, 2, 2]\n", "235 235\n" ] } @@ -689,7 +660,21 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a4488b22b15448bd8473892f256ba528", + "model_id": "032f4a1fb22e4b2f8b0bcfdcf6166191", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/5 [00:00 13\u001b[0m \u001b[38;5;28mprint\u001b[39m(count,\u001b[38;5;28mlen\u001b[39m(\u001b[43mvalidation_tokens_ids\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m))\n\u001b[0;32m 14\u001b[0m count \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m 15\u001b[0m file\u001b[38;5;241m.\u001b[39mwrite(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlist\u001b[39m(ner_vocab\u001b[38;5;241m.\u001b[39mkeys())[\u001b[38;5;28mlist\u001b[39m(ner_vocab\u001b[38;5;241m.\u001b[39mvalues())\u001b[38;5;241m.\u001b[39mindex(el)]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[1;31mIndexError\u001b[0m: list index out of range" ] } ], @@ -1154,24 +1159,134 @@ "count = 0\n", "with open(filename, 'w') as file:\n", " for el in pred:\n", - " if el == 0:\n", + " count+= 1\n", + " if (count == 1):\n", " continue\n", - " else:\n", - " count+= 1\n", - " if count == len(validation_tokens_ids[i]):\n", - " print(count,len(validation_tokens_ids[i]))\n", - " file.write(f\"\\n\")\n", - " i += 1\n", - " \n", - " count = 0\n", - " file.write(f\"{list(ner_vocab.keys())[list(ner_vocab.values()).index(el)]} \")\n", - " " + " if count == (len(validation_tokens_ids[i])):\n", + " i += 1\n", + " file.write(f\"\\n\")\n", + " print(count,len(validation_tokens_ids[i]))\n", + " count = 0\n", + " file.write(f\"{list(ner_vocab.keys())[list(ner_vocab.values()).index(el)]} \")\n", + "\n", + "print(pred[:458]) " + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "3620e4db-ca2b-46ea-b19c-344343d41250", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7c6028fe0def4d5684c733d9c64131ae", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/215 [00:00