This commit is contained in:
bednarco 2021-04-20 18:43:03 +02:00
parent e4adfb04dc
commit 960a201fb5
3 changed files with 620 additions and 0 deletions

View File

@ -0,0 +1,179 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from sklearn.feature_extraction.text import CountVectorizer\n",
"from nltk.tokenize import RegexpTokenizer\n",
"from stop_words import get_stop_words\n",
"from sklearn.model_selection import train_test_split"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [],
"source": [
"data=pd.read_csv('dev-0/in.tsv', sep='\\t', header=None)\n",
"expected_data=pd.read_csv('dev-0/expected.tsv', sep='\\t', header=None)"
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {},
"outputs": [],
"source": [
"data[0] = data[0].str.lower()\n",
"filtered_words = [word for word in data[0] if word not in get_stop_words('polish')]"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"token = RegexpTokenizer(r'[a-zA-Z0-9]+')"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"cv = CountVectorizer(lowercase=True,ngram_range = (1,1),tokenizer = token.tokenize)\n",
"text_counts= cv.fit_transform(data[0])"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<1x5048 sparse matrix of type '<class 'numpy.int64'>'\n",
"\twith 234 stored elements in Compressed Sparse Row format>"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"text_counts"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"X_train, X_test, y_train, y_test = train_test_split(\n",
" text_counts, expected_data[0], test_size=0.3, random_state=1)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"MultinomialNB Accuracy: 0.6296296296296297\n"
]
}
],
"source": [
"from sklearn.naive_bayes import MultinomialNB\n",
"from sklearn import metrics\n",
"clf = MultinomialNB().fit(X_train, y_train)\n",
"predicted= clf.predict(X_test)\n",
"print(\"MultinomialNB Accuracy:\",metrics.accuracy_score(y_test, predicted))"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
"tf=TfidfVectorizer()\n",
"text_tf= tf.fit_transform(filtered_words)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split\n",
"X_train, X_test, y_train, y_test = train_test_split(\n",
" text_tf, expected_data[0], test_size=0.3, random_state=123)"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"MultinomialNB Accuracy: 0.2222222222222222\n"
]
}
],
"source": [
"from sklearn.naive_bayes import MultinomialNB\n",
"from sklearn import metrics\n",
"clf = MultinomialNB().fit(X_train, y_train)\n",
"predicted= clf.predict(X_test)\n",
"print(\"MultinomialNB Accuracy:\",metrics.accuracy_score(y_test, predicted))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

354
Untitled.ipynb Normal file

File diff suppressed because one or more lines are too long

87
dev-0/out.tsv Normal file
View File

@ -0,0 +1,87 @@
1
7
3
9
4
6
2
0
6
3
0
6
7
4
7
2
7
7
3
4
8
4
4
8
0
4
5
4
4
7
2
2
2
4
7
2
7
4
5
9
6
1
2
9
1
3
2
7
5
2
0
3
2
4
1
8
7
7
2
3
2
7
2
2
6
4
2
1
3
2
4
3
1
2
7
0
0
1
9
4
3
0
3
4
2
7
4
1 1
2 7
3 3
4 9
5 4
6 6
7 2
8 0
9 6
10 3
11 0
12 6
13 7
14 4
15 7
16 2
17 7
18 7
19 3
20 4
21 8
22 4
23 4
24 8
25 0
26 4
27 5
28 4
29 4
30 7
31 2
32 2
33 2
34 4
35 7
36 2
37 7
38 4
39 5
40 9
41 6
42 1
43 2
44 9
45 1
46 3
47 2
48 7
49 5
50 2
51 0
52 3
53 2
54 4
55 1
56 8
57 7
58 7
59 2
60 3
61 2
62 7
63 2
64 2
65 6
66 4
67 2
68 1
69 3
70 2
71 4
72 3
73 1
74 2
75 7
76 0
77 0
78 1
79 9
80 4
81 3
82 0
83 3
84 4
85 2
86 7
87 4