s444354/challenging-america-word-gap-prediction
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
4b51db483f
challenging-america-word-ga.../Zad_8.ipynb

37 KiB
Raw Blame History 

import torch
from torch import nn

torch.cuda.empty_cache()
from google.colab import drive
drive.mount('/content/drive')
Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).

%cd drive/MyDrive/moj7
/content/drive/MyDrive/moj7

!ls
config.txt	 in-header.tsv	out-header.tsv	     test-A
dev-0		 model1.bin	processed_train.txt  train
filename.pickle  model2.bin	simplepredict.py     train_new.txt

import pandas as pd
import regex as re
import csv

def clean_text(text):
    text = text.lower().replace('-\\\\\\\\\\\\\\\\n', '').replace('\\\\\\\\\\\\\\\\n', ' ')
    text = re.sub(r'\p{P}', '', text)
    text = text.replace("'t", " not").replace("'s", " is").replace("'ll", " will").replace("'m", " am").replace("'ve", " have")

    return text
train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)

train_data = train_data[[6, 7]]
train_data = pd.concat([train_data, train_labels], axis=1)

train_data['text'] = train_data[6] + train_data[0] + train_data[7]
train_data = train_data[['text']]

with open('processed_train.txt', 'w', encoding='utf-8') as file:
    for _, row in train_data.iterrows():
        text = clean_text(str(row['text']))
        file.write(text + '\n')
<ipython-input-6-c2ca5c6b11cc>:1: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.


  train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
<ipython-input-6-c2ca5c6b11cc>:1: FutureWarning: The warn_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.


  train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
<ipython-input-6-c2ca5c6b11cc>:2: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.


  train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
<ipython-input-6-c2ca5c6b11cc>:2: FutureWarning: The warn_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.


  train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)

import itertools
import lzma
import numpy as np
import regex as re
import torch
import pandas as pd
from torch import nn
from torch.utils.data import IterableDataset, DataLoader
import csv
from itertools import islice, chain
from torchtext.vocab import build_vocab_from_iterator
device='cuda'
train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
train_data = train_data[[6, 7]]
train_data = pd.concat([train_data, train_labels], axis=1)
train_data['text'] = train_data[6] + train_data[0] + train_data[7]
train_data = train_data[['text']]
<ipython-input-9-28a7685109f8>:1: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.


  train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
<ipython-input-9-28a7685109f8>:1: FutureWarning: The warn_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.


  train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
<ipython-input-9-28a7685109f8>:2: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.


  train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
<ipython-input-9-28a7685109f8>:2: FutureWarning: The warn_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.


  train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)

train_data
text
0 came fiom the last place to this\nplace, and t...
1 MB. BOOT'S POLITICAL OBEED\nAttempt to imagine...
2 "Thera were in 1771 only aeventy-nine\n*ub*erl...
3 A gixnl man y nitereRtiiiv dii-clos-\nur«s reg...
4 Tin: 188UB TV THF BBABBT QABJE\nMr. Schiffs *t...
... ...
432017 Sam Clendenin bad a fancy for Ui«\nscience of ...
432018 Wita.htt halting the party ware dilven to the ...
432019 It was the last thing that either of\nthem exp...
432020 settlement with the department.\nIt is also sh...
432021 Flour quotations—low extras at 1 R0®2 50;\ncit...

432022 rows × 1 columns

with open('train_new.txt', 'w', encoding='utf-8') as file:
    for _, row in train_data.iterrows():
        text = clean_text(str(row['text']))
        file.write(text + '\n')


class SimpleTrigramNeuralLanguageModel(nn.Module):
    def __init__(self, vocabulary_size, embedding_size, hidden_size):
        super(SimpleTrigramNeuralLanguageModel, self).__init__()
        self.embedding = nn.Embedding(vocabulary_size * 2, embedding_size)
        self.linear1 = nn.Linear(embedding_size, hidden_size)
        self.linear2 = nn.Linear(hidden_size, vocabulary_size * 2)

    def forward(self, x):
        x = self.embedding(x)
        x = self.linear1(x)
        x = self.linear2(x)
        x = torch.softmax(x, dim=1)
        return x
vocab_size = 38000
embed_size = 300
hidden_size = 256
def words_line(line):
    line = line.rstrip()
    yield '<s>'
    for m in re.finditer(r'[\p{L}0-9\*]+|\p{P}+', line):
        yield m.group(0).lower()
    yield '</s>'

def file_words(file_name):
    with open(file_name, 'r', encoding='utf-8') as fh:
        for line in fh:
            yield words_line(line)
def iterator_look(gen):
    first_prev = None
    sec_prev = None
    for item in gen:
        if first_prev and sec_prev:
            yield (sec_prev+ first_prev, item)
        sec_prev = first_prev
        first_prev = item
class Trigrams(IterableDataset):
    def __init__(self, text_file, vocabulary_size):
        self.vocab = build_vocab_from_iterator(
            file_words(text_file),
            max_tokens = vocabulary_size,
            specials = ['<unk>']
        )
        self.vocab.set_default_index(self.vocab['<unk>'])
        self.vocabulary_size = vocabulary_size
        self.text_file = text_file

    def __iter__(self):
        return iterator_look((self.vocab[t] for t in chain.from_iterable(file_words(self.text_file))))
def training(xx):
  train_dataset_new = Trigrams('train_new.txt', vocab_size)
  model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size, hidden_size).to(device)
  optimizer = torch.optim.Adam(model.parameters())
  criterion = torch.nn.NLLLoss()
  data = DataLoader(train_dataset_new, batch_size=800)
  step = 0
  for epoch in range(1):
      model.train()
      for x, y in data:
          x = x.to(device)
          y = y.to(device)
          optimizer.zero_grad()
          outputs = model(x)
          loss = criterion(torch.log(outputs), y)
          if step % 100 == 0:
              print(step, loss)
          step += 1
          loss.backward()
          optimizer.step()
  torch.save(model.state_dict(), 'model2.bin')
training(xx=0.0001)
0 tensor(11.2670, device='cuda:0', grad_fn=<NllLossBackward0>)
100 tensor(8.0867, device='cuda:0', grad_fn=<NllLossBackward0>)
200 tensor(6.8976, device='cuda:0', grad_fn=<NllLossBackward0>)
300 tensor(6.6515, device='cuda:0', grad_fn=<NllLossBackward0>)
400 tensor(6.6224, device='cuda:0', grad_fn=<NllLossBackward0>)
500 tensor(6.7443, device='cuda:0', grad_fn=<NllLossBackward0>)
600 tensor(6.7064, device='cuda:0', grad_fn=<NllLossBackward0>)
700 tensor(6.8224, device='cuda:0', grad_fn=<NllLossBackward0>)
800 tensor(6.8516, device='cuda:0', grad_fn=<NllLossBackward0>)
900 tensor(6.6103, device='cuda:0', grad_fn=<NllLossBackward0>)
1000 tensor(6.5455, device='cuda:0', grad_fn=<NllLossBackward0>)
1100 tensor(6.8369, device='cuda:0', grad_fn=<NllLossBackward0>)
1200 tensor(6.5587, device='cuda:0', grad_fn=<NllLossBackward0>)
1300 tensor(6.2804, device='cuda:0', grad_fn=<NllLossBackward0>)
1400 tensor(6.5476, device='cuda:0', grad_fn=<NllLossBackward0>)
1500 tensor(6.7563, device='cuda:0', grad_fn=<NllLossBackward0>)
1600 tensor(6.5324, device='cuda:0', grad_fn=<NllLossBackward0>)
1700 tensor(6.6478, device='cuda:0', grad_fn=<NllLossBackward0>)
1800 tensor(6.4025, device='cuda:0', grad_fn=<NllLossBackward0>)
1900 tensor(6.4470, device='cuda:0', grad_fn=<NllLossBackward0>)
2000 tensor(6.8199, device='cuda:0', grad_fn=<NllLossBackward0>)
2100 tensor(6.2291, device='cuda:0', grad_fn=<NllLossBackward0>)
2200 tensor(6.4627, device='cuda:0', grad_fn=<NllLossBackward0>)
2300 tensor(6.5401, device='cuda:0', grad_fn=<NllLossBackward0>)
2400 tensor(6.4382, device='cuda:0', grad_fn=<NllLossBackward0>)
2500 tensor(6.4881, device='cuda:0', grad_fn=<NllLossBackward0>)
2600 tensor(6.2683, device='cuda:0', grad_fn=<NllLossBackward0>)
2700 tensor(6.5393, device='cuda:0', grad_fn=<NllLossBackward0>)
2800 tensor(6.8077, device='cuda:0', grad_fn=<NllLossBackward0>)
2900 tensor(6.6460, device='cuda:0', grad_fn=<NllLossBackward0>)
3000 tensor(6.4482, device='cuda:0', grad_fn=<NllLossBackward0>)
3100 tensor(6.6288, device='cuda:0', grad_fn=<NllLossBackward0>)
3200 tensor(6.4752, device='cuda:0', grad_fn=<NllLossBackward0>)
3300 tensor(6.3716, device='cuda:0', grad_fn=<NllLossBackward0>)
3400 tensor(6.4713, device='cuda:0', grad_fn=<NllLossBackward0>)
3500 tensor(6.4488, device='cuda:0', grad_fn=<NllLossBackward0>)
3600 tensor(6.5300, device='cuda:0', grad_fn=<NllLossBackward0>)
3700 tensor(6.3824, device='cuda:0', grad_fn=<NllLossBackward0>)
3800 tensor(6.6311, device='cuda:0', grad_fn=<NllLossBackward0>)
3900 tensor(6.3778, device='cuda:0', grad_fn=<NllLossBackward0>)
4000 tensor(6.4160, device='cuda:0', grad_fn=<NllLossBackward0>)
4100 tensor(6.5501, device='cuda:0', grad_fn=<NllLossBackward0>)
4200 tensor(6.6891, device='cuda:0', grad_fn=<NllLossBackward0>)
4300 tensor(6.4745, device='cuda:0', grad_fn=<NllLossBackward0>)
4400 tensor(6.7940, device='cuda:0', grad_fn=<NllLossBackward0>)
4500 tensor(6.2111, device='cuda:0', grad_fn=<NllLossBackward0>)
4600 tensor(6.7691, device='cuda:0', grad_fn=<NllLossBackward0>)
4700 tensor(6.2466, device='cuda:0', grad_fn=<NllLossBackward0>)
4800 tensor(6.5852, device='cuda:0', grad_fn=<NllLossBackward0>)
4900 tensor(6.1048, device='cuda:0', grad_fn=<NllLossBackward0>)
5000 tensor(6.5077, device='cuda:0', grad_fn=<NllLossBackward0>)
5100 tensor(6.6974, device='cuda:0', grad_fn=<NllLossBackward0>)
5200 tensor(6.4872, device='cuda:0', grad_fn=<NllLossBackward0>)
5300 tensor(6.4792, device='cuda:0', grad_fn=<NllLossBackward0>)
5400 tensor(6.4319, device='cuda:0', grad_fn=<NllLossBackward0>)
5500 tensor(6.4370, device='cuda:0', grad_fn=<NllLossBackward0>)
5600 tensor(6.5948, device='cuda:0', grad_fn=<NllLossBackward0>)
5700 tensor(6.5184, device='cuda:0', grad_fn=<NllLossBackward0>)
5800 tensor(6.4193, device='cuda:0', grad_fn=<NllLossBackward0>)
5900 tensor(6.4801, device='cuda:0', grad_fn=<NllLossBackward0>)
6000 tensor(6.4735, device='cuda:0', grad_fn=<NllLossBackward0>)
6100 tensor(6.4440, device='cuda:0', grad_fn=<NllLossBackward0>)
6200 tensor(6.3385, device='cuda:0', grad_fn=<NllLossBackward0>)
6300 tensor(6.2252, device='cuda:0', grad_fn=<NllLossBackward0>)
6400 tensor(6.2866, device='cuda:0', grad_fn=<NllLossBackward0>)
6500 tensor(6.8166, device='cuda:0', grad_fn=<NllLossBackward0>)
6600 tensor(6.4074, device='cuda:0', grad_fn=<NllLossBackward0>)
6700 tensor(6.6818, device='cuda:0', grad_fn=<NllLossBackward0>)
6800 tensor(5.9832, device='cuda:0', grad_fn=<NllLossBackward0>)
6900 tensor(6.1267, device='cuda:0', grad_fn=<NllLossBackward0>)
7000 tensor(6.6872, device='cuda:0', grad_fn=<NllLossBackward0>)
7100 tensor(6.4554, device='cuda:0', grad_fn=<NllLossBackward0>)
7200 tensor(6.5397, device='cuda:0', grad_fn=<NllLossBackward0>)
7300 tensor(6.3267, device='cuda:0', grad_fn=<NllLossBackward0>)
7400 tensor(6.4830, device='cuda:0', grad_fn=<NllLossBackward0>)
7500 tensor(6.5805, device='cuda:0', grad_fn=<NllLossBackward0>)
7600 tensor(6.1212, device='cuda:0', grad_fn=<NllLossBackward0>)
7700 tensor(6.2900, device='cuda:0', grad_fn=<NllLossBackward0>)
7800 tensor(6.1379, device='cuda:0', grad_fn=<NllLossBackward0>)
7900 tensor(6.1837, device='cuda:0', grad_fn=<NllLossBackward0>)
8000 tensor(6.5634, device='cuda:0', grad_fn=<NllLossBackward0>)
8100 tensor(6.5012, device='cuda:0', grad_fn=<NllLossBackward0>)
8200 tensor(6.3135, device='cuda:0', grad_fn=<NllLossBackward0>)
8300 tensor(6.6141, device='cuda:0', grad_fn=<NllLossBackward0>)
8400 tensor(6.4679, device='cuda:0', grad_fn=<NllLossBackward0>)
8500 tensor(6.2488, device='cuda:0', grad_fn=<NllLossBackward0>)
8600 tensor(6.3222, device='cuda:0', grad_fn=<NllLossBackward0>)
8700 tensor(6.4057, device='cuda:0', grad_fn=<NllLossBackward0>)
8800 tensor(6.2209, device='cuda:0', grad_fn=<NllLossBackward0>)
8900 tensor(6.6274, device='cuda:0', grad_fn=<NllLossBackward0>)
9000 tensor(6.4992, device='cuda:0', grad_fn=<NllLossBackward0>)
9100 tensor(6.5748, device='cuda:0', grad_fn=<NllLossBackward0>)
9200 tensor(6.2457, device='cuda:0', grad_fn=<NllLossBackward0>)
9300 tensor(6.4364, device='cuda:0', grad_fn=<NllLossBackward0>)
9400 tensor(6.4908, device='cuda:0', grad_fn=<NllLossBackward0>)
9500 tensor(6.5462, device='cuda:0', grad_fn=<NllLossBackward0>)
9600 tensor(6.3248, device='cuda:0', grad_fn=<NllLossBackward0>)
9700 tensor(6.3758, device='cuda:0', grad_fn=<NllLossBackward0>)
9800 tensor(6.1925, device='cuda:0', grad_fn=<NllLossBackward0>)
9900 tensor(6.5854, device='cuda:0', grad_fn=<NllLossBackward0>)
10000 tensor(6.5270, device='cuda:0', grad_fn=<NllLossBackward0>)
10100 tensor(6.3718, device='cuda:0', grad_fn=<NllLossBackward0>)
10200 tensor(6.6314, device='cuda:0', grad_fn=<NllLossBackward0>)
10300 tensor(6.3025, device='cuda:0', grad_fn=<NllLossBackward0>)
10400 tensor(6.2880, device='cuda:0', grad_fn=<NllLossBackward0>)
10500 tensor(6.6817, device='cuda:0', grad_fn=<NllLossBackward0>)
10600 tensor(6.4151, device='cuda:0', grad_fn=<NllLossBackward0>)
10700 tensor(6.5276, device='cuda:0', grad_fn=<NllLossBackward0>)
10800 tensor(6.6714, device='cuda:0', grad_fn=<NllLossBackward0>)
10900 tensor(6.4049, device='cuda:0', grad_fn=<NllLossBackward0>)
11000 tensor(6.2844, device='cuda:0', grad_fn=<NllLossBackward0>)
11100 tensor(6.3522, device='cuda:0', grad_fn=<NllLossBackward0>)
11200 tensor(6.5579, device='cuda:0', grad_fn=<NllLossBackward0>)
11300 tensor(6.6415, device='cuda:0', grad_fn=<NllLossBackward0>)
11400 tensor(6.2489, device='cuda:0', grad_fn=<NllLossBackward0>)
11500 tensor(6.1745, device='cuda:0', grad_fn=<NllLossBackward0>)
11600 tensor(6.5829, device='cuda:0', grad_fn=<NllLossBackward0>)
11700 tensor(6.4514, device='cuda:0', grad_fn=<NllLossBackward0>)
11800 tensor(6.4100, device='cuda:0', grad_fn=<NllLossBackward0>)
11900 tensor(6.2816, device='cuda:0', grad_fn=<NllLossBackward0>)
12000 tensor(6.4974, device='cuda:0', grad_fn=<NllLossBackward0>)
12100 tensor(6.3546, device='cuda:0', grad_fn=<NllLossBackward0>)
12200 tensor(6.4354, device='cuda:0', grad_fn=<NllLossBackward0>)
12300 tensor(6.2498, device='cuda:0', grad_fn=<NllLossBackward0>)
12400 tensor(6.2456, device='cuda:0', grad_fn=<NllLossBackward0>)
12500 tensor(6.2744, device='cuda:0', grad_fn=<NllLossBackward0>)
12600 tensor(6.3540, device='cuda:0', grad_fn=<NllLossBackward0>)
12700 tensor(6.4590, device='cuda:0', grad_fn=<NllLossBackward0>)
12800 tensor(6.3227, device='cuda:0', grad_fn=<NllLossBackward0>)
12900 tensor(6.2072, device='cuda:0', grad_fn=<NllLossBackward0>)
13000 tensor(6.1667, device='cuda:0', grad_fn=<NllLossBackward0>)
13100 tensor(6.4865, device='cuda:0', grad_fn=<NllLossBackward0>)

model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size, hidden_size).to(device)
model.load_state_dict(torch.load('model2.bin'))
model.eval()
train_dataset_new = Trigrams('train_new.txt', vocab_size)

def predict_words(words):
    ixs = torch.tensor(train_dataset_new.vocab.forward(['with'])).to(device)
    predictions = model(ixs)
    total_prob = 0.0
    prediction = ''
    top = torch.topk(predictions[0], 30)
    top_indices = top.indices.tolist()
    top_probs = top.values.tolist()
    top_words = train_dataset_new.vocab.lookup_tokens(top_indices)
    top_preds = list(zip(top_words, top_indices, top_probs))

    for word, _, prob in top_preds:
        if word != '<unk>':
            prediction += f'{word}:{prob} '
            total_prob += prob
    prediction += f':{1 - total_prob}'
    return prediction
model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size, hidden_size).to(device)
model.load_state_dict(torch.load('model2.bin'))
model.eval() 
with lzma.open(f'dev-0/in.tsv.xz', mode='rt', encoding='utf-8') as fid:
      with open(f'dev-0/out-HIDDEN-SIZE={hidden_size}.tsv', 'w', encoding='utf-8', newline='\n') as f:
          for line in fid:
              separated = line.split('\t')
              prefix = separated[6].replace(r'\n', ' ').split()[-2:]
              output_line = predict_words(prefix)
              f.write(output_line + '\n')
with lzma.open(f'test-A/in.tsv.xz', mode='rt', encoding='utf-8') as fid:
      with open(f'test-A/out-HIDDEN-SIZE={hidden_size}.tsv', 'w', encoding='utf-8', newline='\n') as f:
          for line in fid:
              separated = line.split('\t')
              prefix = separated[6].replace(r'\n', ' ').split()[-2:]
              output_line = predict_words(prefix)
              f.write(output_line + '\n')
torch.save(model.state_dict(), 'model2.bin')