Created vocabulary

code_regression.py

@@ -4,19 +4,31 @@ import pickle
 import re
 
 vocabulary=[]
-file_to_save=open("test.tsv","wb")
+#word_to_index_mapping={}
+#index_to_word_mapping={}
 
-def define_vocabulary(file_to_learn_new_words):
-    with open(file_to_learn_new_words,encoding='utf-8') as file:
-        for line in file:
-            #for word in re.findall(r"([a-zA-Z\-]+)", line):
-            for word in line.split():
-                vocabulary.append(word)
-    return vocabulary
+file_to_save=open("test.tsv","w",encoding='utf-8')
+def define_vocabulary(file_to_learn_new_words,expected_path):
+    word_counts = {'paranormal': defaultdict(int), 'skeptic': defaultdict(int)}
+    with open(file_to_learn_new_words, encoding='utf-8') as in_file, open(expected_path, encoding='utf-8')  as  expected_file:
+        for line, exp in zip(in_file, expected_file):
+            class_ = exp.rstrip('\n').replace(' ', '')
+            text, timestamp = line.rstrip('\n').split('\t')
+            tokens = text.lower().split(' ')
+            for token in tokens:
+                if class_ == 'P':
+                    word_counts['paranormal'][token] += 1
+                elif class_ == 'S':
+                    word_counts['skeptic'][token] += 1
+    return word_counts
 
 def main():
-    vocabulary=define_vocabulary('train/in.tsv')
-    file_to_save=vocabulary
+    vocabulary=define_vocabulary('train/in.tsv','train/expected.tsv')
+    ix=1
+    #for word in vocabulary:
+        #word_to_index_mapping(word)=ix
+        #index_to_word_mapping(ix)=word
+
 
 main()