s444465/projekt-glebokie
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

roberta with leaky relu

Browse Source
This commit is contained in:
s444501 2023-02-12 21:21:21 +01:00
parent e9b32f02c9
commit ce785cdd7a
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
roberta.py
View File

@ -53,7 +53,7 @@ class RobertaForSequenceClassificationCustomSimple(RobertaForSequenceClassificat
# Version with custom forward 1 # # Version with custom forward 1 #
####### EDITED #######
class RobertaClassificationHeadCustom(nn.Module): class RobertaClassificationHeadCustom(nn.Module):
"""Head for sentence-level classification tasks.""" """Head for sentence-level classification tasks."""
@ -70,6 +70,7 @@ class RobertaClassificationHeadCustom(nn.Module):
config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
) )
self.dropout = nn.Dropout(classifier_dropout) self.dropout = nn.Dropout(classifier_dropout)
self.leaky_relu = nn.LeakyReLU()
self.out_proj = nn.Linear(hidden_size, config.num_labels) self.out_proj = nn.Linear(hidden_size, config.num_labels)
def forward(self, features, **kwargs): def forward(self, features, **kwargs):
@ -98,17 +99,16 @@ class RobertaClassificationHeadCustom(nn.Module):
) )
x = self.dense_1(x) x = self.dense_1(x)
x = torch.relu(x) x = self.leaky_relu(x)
x = self.dropout(x) x = self.dropout(x)
x = self.dense_2(x) x = self.dense_2(x)
x = torch.relu(x) x = self.leaky_relu(x)
x = self.dropout(x) x = self.dropout(x)
x = self.out_proj(x) x = self.out_proj(x)
return x return x
class RobertaForSequenceClassificationCustom(RobertaForSequenceClassification): class RobertaForSequenceClassificationCustom(RobertaForSequenceClassification):
_keys_to_ignore_on_load_missing = [r"position_ids"] _keys_to_ignore_on_load_missing = [r"position_ids"]
@ -118,6 +118,7 @@ class RobertaForSequenceClassificationCustom(RobertaForSequenceClassification):
self.config = config self.config = config
self.roberta = RobertaModel(config, add_pooling_layer=False) self.roberta = RobertaModel(config, add_pooling_layer=False)
####### EDITED #######
self.classifier = RobertaClassificationHeadCustom(config) self.classifier = RobertaClassificationHeadCustom(config)
# Initialize weights and apply final processing # Initialize weights and apply final processing