Bigger vocab

2022-05-07 16:39:12 +02:00 · 2022-05-07 16:39:12 +02:00 · e581a3667e
commit e581a3667e
parent 9c381a9eea
3 changed files with 17935 additions and 17935 deletions
--- a/dev-0/out.tsv
+++ b/dev-0/out.tsv
--- a/run.py
+++ b/run.py
@ -47,7 +47,7 @@ def get_words_lines_from_file(file_path):
            #     break


-vocab_size = 30000
+vocab_size = 40000

 vocab = build_vocab_from_iterator(
    get_words_lines_from_file('train/in.tsv.xz'),
@ -97,7 +97,7 @@ class Bigrams(IterableDataset):


 def train():
-    batch_size = 15000
+    batch_size = 10000

    train_dataset = Bigrams('train/in.tsv.xz', vocab_size)

--- a/test-A/out.tsv
+++ b/test-A/out.tsv