laptop commit fixed naive baise

.idea/.gitignore

@@ -0,0 +1,2 @@
+# Default ignored files
+/workspace.xml

.idea/Pierwsze.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.8" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="JavaScriptSettings">
+    <option name="languageLevel" value="ES6" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/Pierwsze.iml" filepath="$PROJECT_DIR$/.idea/Pierwsze.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

code.py

@@ -1,15 +1,21 @@
 from collections import defaultdict
 import math
 import pickle
+import re
 
+open_file=('test-A/out.tsv')
+
+#---------------TRAIN START
+
+#Prawdopodobienstwo wylosowania dokumentu
 def calc_class_logprob(expected_path):
     paranormal_classcount=0
     skeptic_classcount=0
-    with open(expected_path) as f:
+    with open(expected_path,encoding='utf-8') as f:
         for line in f:
-            if 'P' in line:
+            if '1' in line:
                 paranormal_classcount += 1
-            if 'S' in line:
+            if '0' in line:
                 skeptic_classcount += 1
 
     paranormal_prob = paranormal_classcount / (paranormal_classcount + skeptic_classcount)
@@ -20,15 +26,25 @@ def calc_class_logprob(expected_path):
 
 def calc_word_count(in_path, expected_path):
     word_counts = {'paranormal':defaultdict(int), 'skeptic': defaultdict(int)}
-    with open(in_path) as in_file, open(expected_path)  as  expected_file:
+    with open(in_path,encoding='utf-8') as in_file, open(expected_path,encoding='utf-8')  as  expected_file:
         for line, exp in zip(in_file, expected_file):
             class_ = exp.rstrip('\n').replace(' ','')
             text, timestamp = line.rstrip('\n').split('\t')
-            tokens = text.lower().split(' ')
+            text = text.lower()
+            text = re.sub(r'\\n+', " ", text)
+            text = re.sub(r'http\S+', " ", text)
+            text = re.sub(r'\/[a-z]\/', " ", text)
+            text = re.sub(r'[^a-z]', " ", text)
+            text = re.sub(r'\s{2,}', " ", text)
+            text = re.sub(r'\W\w{1,3}\W|\A\w{1,3}\W', " ", text)
+            text = re.sub(r'\W\w{1,3}\W|\A\w{1,3}\W', " ", text)
+            text = re.sub(r'\W\w{1,3}\W|\A\w{1,3}\W', " ", text)
+            text = re.sub(r'^\s', "", text)
+            tokens = text.split(' ')
             for token in tokens:
-                if class_ == 'P':
+                if class_ == '1':
                     word_counts['paranormal'][token] += 1
-                elif class_ == 'S':
+                elif class_ == '0':
                     word_counts['skeptic'][token] += 1
     return word_counts
 
@@ -46,12 +62,14 @@ def calc_word_logprobs(word_counts):
             word_logprobs[class_][token] = math.log(word_prob)
     return word_logprobs
 
+#--------------- TRAIN END
 
 def main():
-    paranomal_class_logprob, skeptic_class_logprob = calc_class_logprob("F:/UAM/SEMESTR_I_MGR/SYSTEMY_INTELIGENTNE/ic4g/train/expected.tsv")
+    paranomal_class_logprob, skeptic_class_logprob = calc_class_logprob("train/expected.tsv")
-    word_counts=calc_word_count("F:/UAM/SEMESTR_I_MGR/SYSTEMY_INTELIGENTNE/ic4g/train/in.tsv","F:/UAM/SEMESTR_I_MGR/SYSTEMY_INTELIGENTNE/ic4g/train/expected.tsv")
+    word_counts=calc_word_count("train/in.tsv","train/expected.tsv")
+    word_counts['paranormal'][''] = 0
+    word_counts['skeptic'][''] = 0
     word_logprobs = calc_word_logprobs(word_counts)
     pickle.dump([paranomal_class_logprob, skeptic_class_logprob, word_logprobs], open('naive_base_model.pkl','wb'))
 
-
 main()

code_prediction.py

@@ -0,0 +1,46 @@
+from collections import defaultdict
+import math
+import pickle
+import re
+
+open_file = open('naive_base_model.pkl', 'rb')
+pickle_loaded = pickle.load(open_file)
+paranomal_class_logprob, skeptic_class_logprob, word_logprobs = pickle_loaded
+def prediction(input,output):
+    output_file = open(output,'w')
+    with open(input,encoding='utf-8') as in_file:
+        for line in in_file:
+            temp_paranormal_logprob = paranomal_class_logprob
+            temp_skeptic_logprob = skeptic_class_logprob
+            text, timestamp = line.rstrip('\n').split('\t')
+            text = text.lower()
+            text = re.sub(r'\\n+', " ", text)
+            text = re.sub(r'http\S+', " ", text)
+            text = re.sub(r'\/[a-z]\/', " ", text)
+            text = re.sub(r'[^a-z]', " ", text)
+            text = re.sub(r'\s{2,}', " ", text)
+            text = re.sub(r'\W\w{1,3}\W|\A\w{1,3}\W', " ", text)
+            text = re.sub(r'\W\w{1,3}\W|\A\w{1,3}\W', " ", text)
+            text = re.sub(r'\W\w{1,3}\W|\A\w{1,3}\W', " ", text)
+            text = re.sub(r'^\s', "", text)
+            tokens = text.split(' ')
+            for token in tokens:
+                if token not in word_logprobs['paranormal']:
+                    word_logprobs['paranormal'][token] = -14.78
+                if token not in word_logprobs['skeptic']:
+                    word_logprobs['skeptic'][token] = -15.6
+
+                temp_paranormal_logprob += paranomal_class_logprob + word_logprobs['paranormal'][token]
+                temp_skeptic_logprob += skeptic_class_logprob + word_logprobs['skeptic'][token]
+
+            if temp_paranormal_logprob > temp_skeptic_logprob:
+                output_file.write('1\n')
+            else:
+                output_file.write('0\n')
+##
+def main():
+    prediction('dev-0/in.tsv','dev-0/out.tsv')
+    prediction('test-A/in.tsv/in.tsv','test-A/out.tsv')
+
+main()
+

out_y_hat.tsv

@@ -0,0 +1,6 @@
+Y_hat: -0.0005393564509196473
+Y_hat: -3.9997270479357687
+Y_hat: -4.058054620846149
+Y_hat: -4.948997989328446
+Y_hat: -5.607522681904628
+Y_hat: -5.510496552820199

test_a.tsv

@@ -0,0 +1,6 @@
+0
+0
+0
+0
+0
+0