Custom GPT2 head - WIP

2023-02-12 19:04:16 +01:00 · 2023-02-12 19:04:16 +01:00 · 73cb2a719b
commit 73cb2a719b
parent 3037991562
1 changed files with 4 additions and 0 deletions
--- a/gpt2.py
+++ b/gpt2.py
@ -84,6 +84,10 @@ class GPT2ClassificationHeadCustom(nn.Module):
        x = torch.relu(x)
        x = self.dropout(x)
        
+        x = torch.relu(x)
+        x = self.dense_2(x)
+        x = self.dropout(x)
+
        x = self.out_proj(x)
        return x