diff --git a/.gitignore b/.gitignore index fc96cd3..5f1323b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vscode __pycache__ data/ -results/ \ No newline at end of file +results/ +sacred/ \ No newline at end of file diff --git a/Jenkinsfile-training b/Jenkinsfile-training index 285e3bc..5d6b9c5 100644 --- a/Jenkinsfile-training +++ b/Jenkinsfile-training @@ -59,9 +59,11 @@ node { stage("Docker: Running training model") { sh 'mkdir -p ./.cache' + sh 'mkdir -p ./sacred' // sh "" sh 'python ./src/main.py --train --lr=$LR --batch=$BATCH_SIZE --epochs=$NUM_EPOCHS' sh "cp ./results/model.pt ${WORKSPACE}" + sh "cp ./results/sacred-artifacts.zip ${WORKSPACE}" } } @@ -72,7 +74,7 @@ node { stage('Saving artefacts') { echo 'Goodbye!' sh 'ls' - archiveArtifacts artifacts: '*.pt' + archiveArtifacts artifacts: '*.pt, *.zip' } stage('Triggering eval job') { build job: "s424714-evaluation/master", wait: false diff --git a/src/main.py b/src/main.py index 49949b8..508fd8c 100644 --- a/src/main.py +++ b/src/main.py @@ -1,11 +1,14 @@ import argparse -import torch -from models import BertClassifier, utils -from datasets import NewsDataset -from train import train -from sacred import Experiment -from evaluate import evaluate +import shutil +import torch +from sacred.observers import FileStorageObserver, MongoObserver + +from datasets import NewsDataset +from evaluate import evaluate +from models import BertClassifier, utils +from sacred import Experiment +from train import train # argument parser parser = argparse.ArgumentParser( @@ -26,7 +29,13 @@ parser.add_argument("--num_epochs", "--epochs", "-e", type=int, default=3) # sacred stuff -ex = Experiment("s424714") +ex = Experiment(name="s424714", interactive=True) +SACRED_DIR_PATH = "./sacred" +if not torch.cuda.is_available(): + ex.observers.append(MongoObserver(url="mongodb://admin:IUM_2021@172.17.0.1:27017", db_name="sacred")) +# ex.observers.append(MongoObserver(url="mongodb://admin:IUM_2021@172.17.0.1:27017", db_name="sacred")) +ex.observers.append(FileStorageObserver(SACRED_DIR_PATH)) + ex.add_source_file("./src/train.py") @@ -111,3 +120,4 @@ def main(_run): if __name__ == "__main__": ex.run() + shutil.make_archive(base_name="./results/sacred-artifacts", format="zip", root_dir=SACRED_DIR_PATH)