feat: update jeknkins for sacred training with mongo
This commit is contained in:
parent
34f3b4764d
commit
261c5ca6e2
1
.gitignore
vendored
1
.gitignore
vendored
@ -2,3 +2,4 @@
|
||||
__pycache__
|
||||
data/
|
||||
results/
|
||||
sacred/
|
@ -59,9 +59,11 @@ node {
|
||||
stage("Docker: Running training model")
|
||||
{
|
||||
sh 'mkdir -p ./.cache'
|
||||
sh 'mkdir -p ./sacred'
|
||||
// sh ""
|
||||
sh 'python ./src/main.py --train --lr=$LR --batch=$BATCH_SIZE --epochs=$NUM_EPOCHS'
|
||||
sh "cp ./results/model.pt ${WORKSPACE}"
|
||||
sh "cp ./results/sacred-artifacts.zip ${WORKSPACE}"
|
||||
}
|
||||
|
||||
}
|
||||
@ -72,7 +74,7 @@ node {
|
||||
stage('Saving artefacts') {
|
||||
echo 'Goodbye!'
|
||||
sh 'ls'
|
||||
archiveArtifacts artifacts: '*.pt'
|
||||
archiveArtifacts artifacts: '*.pt, *.zip'
|
||||
}
|
||||
stage('Triggering eval job') {
|
||||
build job: "s424714-evaluation/master", wait: false
|
||||
|
24
src/main.py
24
src/main.py
@ -1,11 +1,14 @@
|
||||
import argparse
|
||||
import torch
|
||||
from models import BertClassifier, utils
|
||||
from datasets import NewsDataset
|
||||
from train import train
|
||||
from sacred import Experiment
|
||||
from evaluate import evaluate
|
||||
import shutil
|
||||
|
||||
import torch
|
||||
from sacred.observers import FileStorageObserver, MongoObserver
|
||||
|
||||
from datasets import NewsDataset
|
||||
from evaluate import evaluate
|
||||
from models import BertClassifier, utils
|
||||
from sacred import Experiment
|
||||
from train import train
|
||||
|
||||
# argument parser
|
||||
parser = argparse.ArgumentParser(
|
||||
@ -26,7 +29,13 @@ parser.add_argument("--num_epochs", "--epochs", "-e", type=int, default=3)
|
||||
|
||||
|
||||
# sacred stuff
|
||||
ex = Experiment("s424714")
|
||||
ex = Experiment(name="s424714", interactive=True)
|
||||
SACRED_DIR_PATH = "./sacred"
|
||||
if not torch.cuda.is_available():
|
||||
ex.observers.append(MongoObserver(url="mongodb://admin:IUM_2021@172.17.0.1:27017", db_name="sacred"))
|
||||
# ex.observers.append(MongoObserver(url="mongodb://admin:IUM_2021@172.17.0.1:27017", db_name="sacred"))
|
||||
ex.observers.append(FileStorageObserver(SACRED_DIR_PATH))
|
||||
|
||||
ex.add_source_file("./src/train.py")
|
||||
|
||||
|
||||
@ -111,3 +120,4 @@ def main(_run):
|
||||
|
||||
if __name__ == "__main__":
|
||||
ex.run()
|
||||
shutil.make_archive(base_name="./results/sacred-artifacts", format="zip", root_dir=SACRED_DIR_PATH)
|
||||
|
Loading…
Reference in New Issue
Block a user