roberta custom head test6

2023-02-12 20:09:44 +01:00 · 2023-02-12 20:09:44 +01:00 · 930d8c7d71
commit 930d8c7d71
parent 9da7c43ad1
1 changed files with 2 additions and 2 deletions
--- a/roberta.py
+++ b/roberta.py
@ -52,11 +52,11 @@ class RobertaLeakyHead(nn.Module):
                )

        x = self.dense_1(x)
-        x = self.leaky_relu(x)
+        x = torch.relu(x)
        x = self.dropout(x)

        x = self.dense_2(x)
-        x = self.leaky_relu(x)
+        x = torch.relu(x)
        x = self.dropout(x)

        x = self.out_proj(x)