35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
from collections import defaultdict
|
|
import math
|
|
import pickle
|
|
import re
|
|
|
|
vocabulary=[]
|
|
#word_to_index_mapping={}
|
|
#index_to_word_mapping={}
|
|
|
|
file_to_save=open("test.tsv","w",encoding='utf-8')
|
|
def define_vocabulary(file_to_learn_new_words,expected_path):
|
|
word_counts = {'paranormal': defaultdict(int), 'skeptic': defaultdict(int)}
|
|
with open(file_to_learn_new_words, encoding='utf-8') as in_file, open(expected_path, encoding='utf-8') as expected_file:
|
|
for line, exp in zip(in_file, expected_file):
|
|
class_ = exp.rstrip('\n').replace(' ', '')
|
|
text, timestamp = line.rstrip('\n').split('\t')
|
|
tokens = text.lower().split(' ')
|
|
for token in tokens:
|
|
if class_ == 'P':
|
|
word_counts['paranormal'][token] += 1
|
|
elif class_ == 'S':
|
|
word_counts['skeptic'][token] += 1
|
|
return word_counts
|
|
|
|
def main():
|
|
vocabulary=define_vocabulary('train/in.tsv','train/expected.tsv')
|
|
ix=1
|
|
#for word in vocabulary:
|
|
#word_to_index_mapping(word)=ix
|
|
#index_to_word_mapping(ix)=word
|
|
|
|
|
|
main()
|
|
|