s430693/paranormal-or-skeptic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

laptop commit fixed naive baise

Browse Source
This commit is contained in:
Bartosz Ogonowski 2020-05-02 18:16:25 +02:00
parent 7c966a9c00
commit 763ce22710
2 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
code.py
View File

@ -28,10 +28,17 @@ def calc_word_count(in_path, expected_path):
word_counts = {'paranormal':defaultdict(int), 'skeptic': defaultdict(int)}
with open(in_path,encoding='utf-8') as in_file, open(expected_path,encoding='utf-8') as expected_file:
for line, exp in zip(in_file, expected_file):
class_ = exp.rstrip('\n').replace(' ','')
class_ = exp.rstrip('\n')
#.replace(' ','')
text, timestamp = line.rstrip('\n').split('\t')
text = re.sub(r'\\n+', " ", text)
text = re.sub(r'http\S+', " ", text)
text = re.sub(r'\/[a-z]\/', " ", text)
text = re.sub(r'[^a-z]', " ", text)
text = re.sub(r'\s{2,}', " ", text)
text = re.sub(r'(\s+|\\n)', ' ', text)
text = re.sub(r'(https?:|www)\S+(\s+|$)', ' URL ', text)
text = re.sub(r'\W\w{1,3}\W|\A\w{1,3}\W', " ", text)
text = re.sub(r'^\s', "", text)
tokens = text.lower().split(' ')
for token in tokens:
if class_ == 'P':

10
code_prediction.py
View File

@ -11,13 +11,21 @@ paranomal_class_logprob, skeptic_class_logprob, word_logprobs = pickle_loaded
#Niektórych słów nie bezie w zbiorze treningowym dev-0 i dev-A
def prediction(input,output):
output_file = open(output,'w')
pickle_load = pickle.load(open('naive_base_model.pkl', 'rb'))
paranormal_class_logprob, skeptic_class_logprob, word_logprob = pickle_load
with open(input,encoding='utf-8') as in_file:
for line in in_file:
temp_paranormal_logprob = paranomal_class_logprob
temp_skeptic_logprob = skeptic_class_logprob
text, timestamp = line.rstrip('\n').split('\t')
text = re.sub(r'\\n+', " ", text)
text = re.sub(r'http\S+', " ", text)
text = re.sub(r'\/[a-z]\/', " ", text)
text = re.sub(r'[^a-z]', " ", text)
text = re.sub(r'\s{2,}', " ", text)
text = re.sub(r'(\s+|\\n)', ' ', text)
text = re.sub(r'(https?:|www)\S+(\s+|$)', ' URL ', text)
text = re.sub(r'\W\w{1,3}\W|\A\w{1,3}\W', " ", text)
text = re.sub(r'^\s', "", text)
tokens = text.lower().split(' ')
for token in tokens:
if token not in word_logprobs['paranormal']: