From f52c10479cc04458e10fc68376394e0a0ab6e268 Mon Sep 17 00:00:00 2001 From: s444501 Date: Sun, 12 Feb 2023 19:35:22 +0100 Subject: [PATCH] roberta custom head test3 --- roberta.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/roberta.py b/roberta.py index 45575aa..293ac02 100644 --- a/roberta.py +++ b/roberta.py @@ -24,6 +24,7 @@ class RobertaLeakyHead(nn.Module): classifier_dropout = ( config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob ) + self.leaky_relu = nn.LeakyReLU() self.dropout = nn.Dropout(classifier_dropout) self.out_proj = nn.Linear(hidden_size, config.num_labels) @@ -53,19 +54,19 @@ class RobertaLeakyHead(nn.Module): ) x = self.dense_1(x) - x = torch.nn.LeakyReLU(x) + x = self.leaky_relu(x) x = self.dropout(x) x = self.dense_2(x) - x = torch.nn.LeakyReLU(x) + x = self.leaky_relu(x) x = self.dropout(x) x = self.dense_3(x) - x = torch.nn.LeakyReLU(x) + x = self.leaky_relu(x) x = self.dropout(x) x = self.dense_4(x) - x = torch.nn.LeakyReLU(x) + x = self.leaky_relu(x) x = self.dropout(x) x = self.out_proj(x)