hf roberta (linear top layer by hand instead of hf) with learning scheaduler)

hf_roberta_base/03_train_pytorch_regression.py

@@ -42,18 +42,19 @@ eval_dataloader_full = DataLoader(eval_dataset_full, batch_size=BATCH_SIZE)
 
 device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
 model = RobertaModel.from_pretrained('roberta-base')
+#model = RobertaModel(model.config)
 model.regressor_head = RegressorHead(768).to('cuda')
 model.to(device)
 
 optimizer = Adam(model.parameters(), lr=LR)
 
 num_training_steps = NUM_EPOCHS * len(train_dataloader)
-#lr_scheduler = get_scheduler(
-#    "linear",
-#    optimizer=optimizer,
-#    num_warmup_steps=WARMUP_STEPS,
-#    num_training_steps=num_training_steps
-#)
+lr_scheduler = get_scheduler(
+    "linear",
+    optimizer=optimizer,
+    num_warmup_steps=WARMUP_STEPS,
+    num_training_steps=num_training_steps
+)
 
 
 progress_bar = tqdm(range(num_training_steps))
@@ -117,7 +118,7 @@ for epoch in range(NUM_EPOCHS):
         progress_bar.update(1)
 
         optimizer.step()
-        #lr_scheduler.step()
+        lr_scheduler.step()
         optimizer.zero_grad()
         model.zero_grad()
 

hf_roberta_base/config.py

@@ -1,8 +1,8 @@
 #MODEL = '../MODELS/without_date/checkpoint-395000'
 MODEL = 'roberta-base'
-BATCH_SIZE = 50
+BATCH_SIZE = 90
 EARLY_STOPPING = 3
-WARMUP_STEPS = 10_000
+WARMUP_STEPS = 5_000
 LR=1e-6
 NUM_EPOCHS = 20
 STEPS_EVAL = 5_000