change

generate.py

@@ -9,48 +9,50 @@ import numpy as np
 from model import BERT_Arch
 from sklearn.metrics import classification_report
 from sklearn.metrics import accuracy_score
+from transformers import BertTokenizerFast, BertForSequenceClassification
+from transformers import Trainer, TrainingArguments
 
-path = 'saved_weights.pt'
+class Dataset(torch.utils.data.Dataset):
-tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+    def __init__(self, encodings, labels):
-device = torch.device("cuda")
+        self.encodings = encodings
+        self.labels = labels
 
-bert = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased')
+    def __getitem__(self, idx):
+        item = {k: torch.tensor(v[idx]) for k, v in self.encodings.items()}
+        item["labels"] = torch.tensor([self.labels[idx]])
+        return item
 
-model = BERT_Arch(bert)
+    def __len__(self):
-model.load_state_dict(torch.load(path))
+        return len(self.labels)
-model.to(device)
 
-test_data = pd.read_csv("dev-0/in.tsv", sep="\t")
+device = torch.device('cuda')
-test_data.columns = ["text", "d"]
 
-test_target = pd.read_csv("dev-0/expected.tsv", sep="\t")
+train_texts = \
+    pd.read_csv('train/in.tsv.xz', compression='xz', sep='\t', header=None, error_bad_lines=False, quoting=3)[0].tolist()[:1000]
+train_labels = pd.read_csv('train/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist()[:1000]
+dev_texts = pd.read_csv('dev-0/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)[0].tolist()[:1000]
+dev_labels = pd.read_csv('dev-0/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist()[:1000]
+model_name = "bert-base-uncased"
 
-tokens_train = tokenizer.batch_encode_plus(
+model = BertForSequenceClassification.from_pretrained(
-    test_data["text"],
+    model_name, num_labels=len(pd.unique(train_labels))).to(device)
-    max_length = 25,
+max_length = 512
-    padding='max_length',
+tokenizer = BertTokenizerFast.from_pretrained(model_name, do_lower_case=True)
-    truncation=True
-)
 
-test_seq = torch.tensor(tokens_train['input_ids'])
+# model.load_pretrained(model_path)
-test_mask = torch.tensor(tokens_train['attention_mask'])
+# tokenizer.load_pretrainded(model_path)
 
-#define a batch size
+train_encodings = tokenizer(train_texts, truncation=True, padding=True, max_length=max_length)
-batch_size = 32
+valid_encodings = tokenizer(dev_texts, truncation=True, padding=True, max_length=max_length)
 
-# wrap tensors
+input_ids = torch.tensor(valid_encodings.data['input_ids'])[:100]
-test_data = TensorDataset(test_seq, test_mask)
+attention_mask = torch.tensor(valid_encodings.data['attention_mask'])[:100]
-
-# sampler for sampling the data during training
-test_sampler = RandomSampler(test_data)
-
-# dataLoader for train set
-test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)
 
 with torch.no_grad():
-  preds = model(test_seq.to(device), test_mask.to(device))
+  preds = model(input_ids.to(device), attention_mask.to(device))
-  preds = preds.detach().cpu().numpy()
+  preds = preds.logits.detach().cpu().numpy()
   preds = np.argmax(preds, axis = 1)
     
-print(classification_report(test_target['0'], preds))
+print(preds)
-print(accuracy_score(test_target['0'], preds))
+print(classification_report(dev_labels, preds))
+print(accuracy_score(dev_labels, preds))

main.py

@@ -1,215 +1,90 @@
-import pandas as pd
-from transformers import BertTokenizer, AdamW, AutoModelForSequenceClassification
 import torch
-import matplotlib.pyplot as plt
+from transformers.file_utils import is_torch_available
-from torch.utils.data import TensorDataset, DataLoader, RandomSampler
+from transformers import BertTokenizerFast, BertForSequenceClassification
-import torch.nn as nn
+from transformers import Trainer, TrainingArguments
-from sklearn.utils.class_weight import compute_class_weight
 import numpy as np
-from model import BERT_Arch
+import random
-
+from sklearn.metrics import accuracy_score
-train_input_path = "train/in.tsv"
+import pandas as pd
-train_target_path = "train/expected.tsv"
-
-train_input = pd.read_csv(train_input_path, sep="\t")
-train_input.columns=["text", "d"]
-train_target = pd.read_csv(train_target_path, sep="\t")
-
-tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-device = torch.device("cuda")
 
 
-# seq_len = [len(i.split()) for i in train_input["text"]]
+class Dataset(torch.utils.data.Dataset):
+    def __init__(self, encodings, labels):
+        self.encodings = encodings
+        self.labels = labels
 
-# pd.Series(seq_len).hist(bins = 30)
+    def __getitem__(self, idx):
-# plt.show()
+        item = {k: torch.tensor(v[idx]) for k, v in self.encodings.items()}
+        item["labels"] = torch.tensor([self.labels[idx]])
+        return item
 
-bert = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased')
+    def __len__(self):
+        return len(self.labels)
 
 
-tokens_train = tokenizer.batch_encode_plus(
+def set_seed(seed: int):
-    train_input["text"],
+    random.seed(seed)
-    max_length = 25,
+    np.random.seed(seed)
-    padding='max_length',
+    if is_torch_available():
-    truncation=True
+        torch.manual_seed(seed)
+        torch.cuda.manual_seed_all(seed)
+
+def compute_metrics(pred):
+    labels = pred.label_ids
+    preds = pred.predictions.argmax(-1)
+    acc = accuracy_score(labels, preds)
+    return {
+        'accuracy': acc,
+    }
+
+set_seed(1)
+
+train_texts = \
+    pd.read_csv('train/in.tsv.xz', compression='xz', sep='\t', header=None, error_bad_lines=False, quoting=3)[0].tolist()
+train_labels = pd.read_csv('train/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist()
+dev_texts = pd.read_csv('dev-0/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)[0].tolist()
+dev_labels = pd.read_csv('dev-0/expected.tsv', sep='\t', header=None, quoting=3)[0].tolist()
+# test_texts = pd.read_table('test-A/in.tsv.xz', compression='xz', sep='\t', header=None, quoting=3)
+
+model_name = "bert-base-uncased"
+max_length = 25
+tokenizer = BertTokenizerFast.from_pretrained(model_name, do_lower_case=True)
+
+train_encodings = tokenizer(train_texts, truncation=True, padding=True, max_length=max_length)
+valid_encodings = tokenizer(dev_texts, truncation=True, padding=True, max_length=max_length)
+
+train_dataset = Dataset(train_encodings, train_labels)
+valid_dataset = Dataset(valid_encodings, dev_labels)
+
+model = BertForSequenceClassification.from_pretrained(
+    model_name, num_labels=len(pd.unique(train_labels))).to("cuda")
+
+training_args = TrainingArguments(
+    output_dir='./results',          # output directory
+    num_train_epochs=3,              # total number of training epochs
+    per_device_train_batch_size=1,  # batch size per device during training
+    per_device_eval_batch_size=1,   # batch size for evaluation
+    warmup_steps=500,                # number of warmup steps for learning rate scheduler
+    weight_decay=0.01,               # strength of weight decay
+    logging_dir='./logs',            # directory for storing logs
+    load_best_model_at_end=True,     # load the best model when finished training (default metric is loss)
+    # but you can specify `metric_for_best_model` argument to change to accuracy or other metric
+    logging_steps=200,               # log & save weights each logging_steps
+    evaluation_strategy="steps",     # evaluate each `logging_steps`
 )
 
-train_seq = torch.tensor(tokens_train['input_ids'])
+trainer = Trainer(
-train_mask = torch.tensor(tokens_train['attention_mask'])
+    model=model,                         # the instantiated Transformers model to be trained
-train_y = torch.tensor(train_target.to_numpy())
+    args=training_args,                  # training arguments, defined above
+    train_dataset=train_dataset,         # training dataset
+    eval_dataset=valid_dataset,          # evaluation dataset
+    compute_metrics=compute_metrics,     # the callback that computes metrics of interest
+)
 
-#define a batch size
+trainer.train()
-batch_size = 32
 
-# wrap tensors
+trainer.evaluate()
-train_data = TensorDataset(train_seq, train_mask, train_y)
 
-# sampler for sampling the data during training
+model_path = "bert-base-uncased-pretrained"
-train_sampler = RandomSampler(train_data)
+model.save_pretrained(model_path)
-
+tokenizer.save_pretrained(model_path)
-# dataLoader for train set
-train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
-
-for param in bert.parameters():
-    param.requires_grad = False
-
-model = BERT_Arch(bert)
-model = model.to(device)
-# model.cuda(0)
-
-optimizer = AdamW(model.parameters(), lr = 1e-5)
-
-class_weights = compute_class_weight('balanced', np.unique(train_target.to_numpy()), train_target['1'])
-weights= torch.tensor(class_weights,dtype=torch.float)
-weights = weights.to(device)
-
-# define the loss function
-cross_entropy  = nn.NLLLoss(weight=weights) 
-
-# number of training epochs
-epochs = 10
-
-def train():
-  
-  model.train()
-
-  total_loss, total_accuracy = 0, 0
-  
-  # empty list to save model predictions
-  total_preds=[]
-  
-  # iterate over batches
-  for step,batch in enumerate(train_dataloader):
-    
-    # progress update after every 50 batches.
-    if step % 50 == 0 and not step == 0:
-      print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))
-
-    # push the batch to gpu
-    batch = [r.to(device) for r in batch]
- 
-    sent_id, mask, labels = batch
-
-    # clear previously calculated gradients 
-    model.zero_grad()        
-
-    # get model predictions for the current batch
-    preds = model(sent_id, mask)
-
-    # compute the loss between actual and predicted values
-    labels = torch.tensor([x[0] for x in labels]).to(device)
-    loss = cross_entropy(preds, labels)
-
-    # add on to the total loss
-    total_loss = total_loss + loss.item()
-
-    # backward pass to calculate the gradients
-    loss.backward()
-
-    # clip the the gradients to 1.0. It helps in preventing the exploding gradient problem
-    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
-
-    # update parameters
-    optimizer.step()
-
-    # model predictions are stored on GPU. So, push it to CPU
-    preds=preds.detach().cpu().numpy()
-
-    # append the model predictions
-    total_preds.append(preds)
-
-  # compute the training loss of the epoch
-  avg_loss = total_loss / len(train_dataloader)
-  
-  # predictions are in the form of (no. of batches, size of batch, no. of classes).
-  # reshape the predictions in form of (number of samples, no. of classes)
-  total_preds  = np.concatenate(total_preds, axis=0)
-
-  #returns the loss and predictions
-  return avg_loss, total_preds
-
-def evaluate():
-  
-  print("\nEvaluating...")
-  
-  # deactivate dropout layers
-  model.eval()
-
-  total_loss, total_accuracy = 0, 0
-  
-  # empty list to save the model predictions
-  total_preds = []
-
-  # iterate over batches
-  for step,batch in enumerate(train_dataloader):
-    
-    # Progress update every 50 batches.
-    if step % 50 == 0 and not step == 0:
-      
-      # Calculate elapsed time in minutes.
-            
-      # Report progress.
-      print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))
-
-    # push the batch to gpu
-    batch = [t.to(device) for t in batch]
-
-    sent_id, mask, labels = batch
-
-    # deactivate autograd
-    with torch.no_grad():
-      
-      # model predictions
-      preds = model(sent_id, mask)
-
-      # compute the validation loss between actual and predicted values
-      labels = torch.tensor([x[0] for x in labels]).to(device)
-      loss = cross_entropy(preds,labels)
-
-      total_loss = total_loss + loss.item()
-
-      preds = preds.detach().cpu().numpy()
-
-      total_preds.append(preds)
-
-  # compute the validation loss of the epoch
-  avg_loss = total_loss / len(train_dataloader) 
-
-  # reshape the predictions in form of (number of samples, no. of classes)
-  total_preds  = np.concatenate(total_preds, axis=0)
-
-  return avg_loss, total_preds
-
-# avg_loss, total_preds = train()
-# set initial loss to infinite
-best_valid_loss = float('inf')
-
-# empty lists to store training and validation loss of each epoch
-train_losses=[]
-valid_losses=[]
-
-print("Started training!")
-#for each epoch
-for epoch in range(epochs):
-     
-    print('\n Epoch {:} / {:}'.format(epoch + 1, epochs))
-    
-    #train model
-    train_loss, _ = train()
-    
-    #evaluate model
-    valid_loss, _ = evaluate()
-    
-    #save the best model
-    if valid_loss < best_valid_loss:
-        best_valid_loss = valid_loss
-        torch.save(model.state_dict(), 'saved_weights.pt')
-    
-    # append training and validation loss
-    train_losses.append(train_loss)
-    valid_losses.append(valid_loss)
-    
-    print(f'\nTraining Loss: {train_loss:.3f}')
-    print(f'Validation Loss: {valid_loss:.3f}')
-
-print("Finished !!!")

model.py

@@ -1,44 +0,0 @@
-import torch.nn as nn
-
-class BERT_Arch(nn.Module):
-
-    def __init__(self, bert):
-      
-      super(BERT_Arch, self).__init__()
-
-      self.bert = bert 
-      
-      # dropout layer
-      self.dropout = nn.Dropout(0.1)
-      
-      # relu activation function
-      self.relu =  nn.ReLU()
-
-      # dense layer 1
-      self.fc1 = nn.Linear(2,512)
-      
-      # dense layer 2 (Output layer)
-      self.fc2 = nn.Linear(512,2)
-
-      #softmax activation function
-      self.softmax = nn.LogSoftmax(dim=1)
-
-    #define the forward pass
-    def forward(self, sent_id, mask):
-
-      #pass the inputs to the model  
-      senence_classifier_output = self.bert(sent_id, attention_mask=mask)
-      x = senence_classifier_output.logits.float()
-      x = self.fc1(x)
-
-      x = self.relu(x)
-
-      x = self.dropout(x)
-
-      # output layer
-      x = self.fc2(x)
-      
-      # apply softmax activation
-      x = self.softmax(x)
-
-      return x