uczenie-glebokie-w-przetwarzaniu-tekstu/Projekt.ipynb

import torch
from torch.utils.data import Dataset, DataLoader
from transformers import BertTokenizer, BertForSequenceClassification, AdamW
from sklearn.metrics import accuracy_score, classification_report
from datasets import load_dataset

c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\tqdm\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

dataset = load_dataset('dair-ai/emotion')

print(dataset)

c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\datasets\load.py:1429: FutureWarning: The repository for dair-ai/emotion contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/dair-ai/emotion
You can avoid this message in future by passing the argument `trust_remote_code=True`.
Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.
  warnings.warn(

DatasetDict({
    train: Dataset({
        features: ['text', 'label'],
        num_rows: 16000
    })
    validation: Dataset({
        features: ['text', 'label'],
        num_rows: 2000
    })
    test: Dataset({
        features: ['text', 'label'],
        num_rows: 2000
    })
})

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=6)


max_len = 128 

train_data = dataset['train']
train_encodings = tokenizer(train_data['text'], truncation=True, padding='max_length', max_length=max_len, return_tensors='pt')
train_labels = torch.tensor(train_data['label'])

train_dataset = torch.utils.data.TensorDataset(train_encodings['input_ids'], train_encodings['attention_mask'], train_labels)
train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)

optimizer = AdamW(model.parameters(), lr=2e-5)

criterion = torch.nn.CrossEntropyLoss()

num_epochs = 3
print(model)

Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\transformers\optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
  warnings.warn(

BertForSequenceClassification(
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768, padding_idx=0)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
      (dropout): Dropout(p=0.1, inplace=False)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0-11): 12 x BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
            (intermediate_act_fn): GELUActivation()
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (dropout): Dropout(p=0.1, inplace=False)
  (classifier): Linear(in_features=768, out_features=6, bias=True)
)

for epoch in range(num_epochs):
    model.train()
    total_loss = 0.0

    for input_ids, attention_mask, labels in train_loader:
        optimizer.zero_grad()

        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
        loss = outputs.loss
        total_loss += loss.item()

        loss.backward()
        optimizer.step()

    average_loss = total_loss / len(train_loader)
    print(f'Epoch {epoch+1}/{num_epochs} - Average Loss: {average_loss}')

model.save_pretrained('emotion_model')

model.eval()

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[4], line 8
      5 for input_ids, attention_mask, labels in train_loader:
      6     optimizer.zero_grad()
----> 8     outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
      9     loss = outputs.loss
     10     total_loss += loss.item()

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\transformers\models\bert\modeling_bert.py:1564, in BertForSequenceClassification.forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
   1556 r"""
   1557 labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
   1558     Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
   1559     config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
   1560     `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
   1561 """
   1562 return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-> 1564 outputs = self.bert(
   1565     input_ids,
   1566     attention_mask=attention_mask,
   1567     token_type_ids=token_type_ids,
   1568     position_ids=position_ids,
   1569     head_mask=head_mask,
   1570     inputs_embeds=inputs_embeds,
   1571     output_attentions=output_attentions,
   1572     output_hidden_states=output_hidden_states,
   1573     return_dict=return_dict,
   1574 )
   1576 pooled_output = outputs[1]
   1578 pooled_output = self.dropout(pooled_output)

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\transformers\models\bert\modeling_bert.py:1013, in BertModel.forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
   1004 head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
   1006 embedding_output = self.embeddings(
   1007     input_ids=input_ids,
   1008     position_ids=position_ids,
   (...)
   1011     past_key_values_length=past_key_values_length,
   1012 )
-> 1013 encoder_outputs = self.encoder(
   1014     embedding_output,
   1015     attention_mask=extended_attention_mask,
   1016     head_mask=head_mask,
   1017     encoder_hidden_states=encoder_hidden_states,
   1018     encoder_attention_mask=encoder_extended_attention_mask,
   1019     past_key_values=past_key_values,
   1020     use_cache=use_cache,
   1021     output_attentions=output_attentions,
   1022     output_hidden_states=output_hidden_states,
   1023     return_dict=return_dict,
   1024 )
   1025 sequence_output = encoder_outputs[0]
   1026 pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\transformers\models\bert\modeling_bert.py:607, in BertEncoder.forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
    596     layer_outputs = self._gradient_checkpointing_func(
    597         layer_module.__call__,
    598         hidden_states,
   (...)
    604         output_attentions,
    605     )
    606 else:
--> 607     layer_outputs = layer_module(
    608         hidden_states,
    609         attention_mask,
    610         layer_head_mask,
    611         encoder_hidden_states,
    612         encoder_attention_mask,
    613         past_key_value,
    614         output_attentions,
    615     )
    617 hidden_states = layer_outputs[0]
    618 if use_cache:

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\transformers\models\bert\modeling_bert.py:497, in BertLayer.forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
    485 def forward(
    486     self,
    487     hidden_states: torch.Tensor,
   (...)
    494 ) -> Tuple[torch.Tensor]:
    495     # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
    496     self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
--> 497     self_attention_outputs = self.attention(
    498         hidden_states,
    499         attention_mask,
    500         head_mask,
    501         output_attentions=output_attentions,
    502         past_key_value=self_attn_past_key_value,
    503     )
    504     attention_output = self_attention_outputs[0]
    506     # if decoder, the last output is tuple of self-attn cache

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\transformers\models\bert\modeling_bert.py:427, in BertAttention.forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
    417 def forward(
    418     self,
    419     hidden_states: torch.Tensor,
   (...)
    425     output_attentions: Optional[bool] = False,
    426 ) -> Tuple[torch.Tensor]:
--> 427     self_outputs = self.self(
    428         hidden_states,
    429         attention_mask,
    430         head_mask,
    431         encoder_hidden_states,
    432         encoder_attention_mask,
    433         past_key_value,
    434         output_attentions,
    435     )
    436     attention_output = self.output(self_outputs[0], hidden_states)
    437     outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\modules\module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\transformers\models\bert\modeling_bert.py:355, in BertSelfAttention.forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
    352     attention_scores = attention_scores + attention_mask
    354 # Normalize the attention scores to probabilities.
--> 355 attention_probs = nn.functional.softmax(attention_scores, dim=-1)
    357 # This is actually dropping out entire tokens to attend to, which might
    358 # seem a bit unusual, but is taken from the original Transformer paper.
    359 attention_probs = self.dropout(attention_probs)

File c:\Users\KamBo\AppData\Local\Programs\Python\Python311\Lib\site-packages\torch\nn\functional.py:1856, in softmax(input, dim, _stacklevel, dtype)
   1854     dim = _get_softmax_dim("softmax", input.dim(), _stacklevel)
   1855 if dtype is None:
-> 1856     ret = input.softmax(dim)
   1857 else:
   1858     ret = input.softmax(dim, dtype=dtype)

KeyboardInterrupt: 

all_labels = []
all_predictions = []

with torch.no_grad():
    for input_ids, attention_mask, labels in train_loader:
        outputs = model(input_ids, attention_mask=attention_mask)
        predictions = torch.argmax(outputs.logits, dim=1)

        all_labels.extend(labels.numpy())
        all_predictions.extend(predictions.numpy())

accuracy = accuracy_score(all_labels, all_predictions)
classification_report_str = classification_report(all_labels, all_predictions)

print(f'Accuracy: {accuracy}')
print('Classification Report:')
print(classification_report_str)

import torch
from torch.utils.data import Dataset, DataLoader
from transformers import GPT2Tokenizer, GPT2ForSequenceClassification, AdamW
from sklearn.metrics import accuracy_score, classification_report
from datasets import load_dataset

dataset = load_dataset('dair-ai/emotion')

print(dataset)

tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2ForSequenceClassification.from_pretrained('gpt2', num_labels=6)  # 6 - liczba klas (0-5)

max_len = 128

train_data = dataset['train']
train_encodings = tokenizer(train_data['text'], truncation=True, padding='max_length', max_length=max_len, return_tensors='pt')
train_labels = torch.tensor(train_data['label'])

train_dataset = torch.utils.data.TensorDataset(train_encodings['input_ids'], train_encodings['attention_mask'], train_labels)
train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)

optimizer = AdamW(model.parameters(), lr=2e-5)

criterion = torch.nn.CrossEntropyLoss()

num_epochs = 3
for epoch in range(num_epochs):
    model.train()
    total_loss = 0.0

    for input_ids, attention_mask, labels in train_loader:
        optimizer.zero_grad()

        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
        loss = outputs.loss
        total_loss += loss.item()

        loss.backward()
        optimizer.step()

    average_loss = total_loss / len(train_loader)
    print(f'Epoch {epoch+1}/{num_epochs} - Average Loss: {average_loss}')

model.save_pretrained('emotion_gpt2_model')

model.eval()
all_labels = []
all_predictions = []

with torch.no_grad():
    for input_ids, attention_mask, labels in train_loader:
        outputs = model(input_ids, attention_mask=attention_mask)
        predictions = torch.argmax(outputs.logits, dim=1)

        all_labels.extend(labels.numpy())
        all_predictions.extend(predictions.numpy())

accuracy = accuracy_score(all_labels, all_predictions)
classification_report_str = classification_report(all_labels, all_predictions)

print(f'Accuracy: {accuracy}')
print('Classification Report:')
print(classification_report_str)

import torch
from torch.utils.data import Dataset, DataLoader
from transformers import T5Tokenizer, T5ForConditionalGeneration, AdamW
from sklearn.metrics import accuracy_score, classification_report
from datasets import load_dataset

dataset = load_dataset('dair-ai/emotion')

print(dataset)

tokenizer = T5Tokenizer.from_pretrained('t5-small')
model = T5ForConditionalGeneration.from_pretrained('t5-small')

max_len = 128

train_data = dataset['train']
train_encodings = tokenizer(train_data['text'], truncation=True, padding='max_length', max_length=max_len, return_tensors='pt')
train_labels = torch.tensor(train_data['label'])

train_dataset = torch.utils.data.TensorDataset(train_encodings['input_ids'], train_encodings['attention_mask'], train_labels)
train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)

optimizer = AdamW(model.parameters(), lr=2e-5)

criterion = torch.nn.CrossEntropyLoss()

num_epochs = 3
for epoch in range(num_epochs):
    model.train()
    total_loss = 0.0

    for input_ids, attention_mask, labels in train_loader:
        optimizer.zero_grad()

        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
        loss = outputs.loss
        total_loss += loss.item()

        loss.backward()
        optimizer.step()

    average_loss = total_loss / len(train_loader)
    print(f'Epoch {epoch+1}/{num_epochs} - Average Loss: {average_loss}')

model.save_pretrained('emotion_t5_model')

model.eval()
all_labels = []
all_predictions = []

with torch.no_grad():
    for input_ids, attention_mask, labels in train_loader:
        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
        predictions = torch.argmax(outputs.logits, dim=1)

        all_labels.extend(labels.numpy())
        all_predictions.extend(predictions.numpy())

accuracy = accuracy_score(all_labels, all_predictions)
classification_report_str = classification_report(all_labels, all_predictions)

print(f'Accuracy: {accuracy}')
print('Classification Report:')
print(classification_report_str)

import torch
from torch.utils.data import Dataset, DataLoader
from transformers import T5Tokenizer, T5ForConditionalGeneration, AdamW
from sklearn.metrics import accuracy_score, classification_report
from datasets import load_dataset

dataset = load_dataset('dair-ai/emotion')

print(dataset)

tokenizer = T5Tokenizer.from_pretrained('google/flan-t5-small')
model = T5ForConditionalGeneration.from_pretrained('google/flan-t5-small')

max_len = 128

train_data = dataset['train']
train_labels = train_data['label']

train_prompts = [f'emotion: {text}' for text in train_data['text']]

train_encodings = tokenizer(train_prompts, truncation=True, padding='max_length', max_length=max_len, return_tensors='pt')
train_labels = torch.tensor(train_labels)

train_dataset = torch.utils.data.TensorDataset(train_encodings['input_ids'], train_encodings['attention_mask'], train_labels)
train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)

optimizer = AdamW(model.parameters(), lr=2e-5)

criterion = torch.nn.CrossEntropyLoss()

num_epochs = 3
for epoch in range(num_epochs):
    model.train()
    total_loss = 0.0

    for input_ids, attention_mask, labels in train_loader:
        optimizer.zero_grad()

        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
        loss = outputs.loss
        total_loss += loss.item()

        loss.backward()
        optimizer.step()

    average_loss = total_loss / len(train_loader)
    print(f'Epoch {epoch+1}/{num_epochs} - Average Loss: {average_loss}')

# Zapisz model
model.save_pretrained('emotion_flant5_model')

# Ewaluacja modelu
model.eval()
all_labels = []
all_predictions = []

with torch.no_grad():
    for input_ids, attention_mask, labels in train_loader:
        outputs = model.generate(input_ids, attention_mask=attention_mask, max_length=1)
        predictions = torch.argmax(outputs, dim=1)

        all_labels.extend(labels.numpy())
        all_predictions.extend(predictions.numpy())

# Oblicz metryki ewaluacyjne
accuracy = accuracy_score(all_labels, all_predictions)
classification_report_str = classification_report(all_labels, all_predictions)

print(f'Accuracy: {accuracy}')
print('Classification Report:')
print(classification_report_str)