s444417/s444417-paranormal-or-skeptic-ISI-public
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
s444417 2022-05-18 10:50:06 +02:00
parent 0fd3ddf4f3
commit 5ad6b81959
3 changed files with 1753 additions and 2199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2472
dev-0/out.tsv
View File

File diff suppressed because it is too large Load Diff

80
run.ipynb
View File

@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 100,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@ -16,12 +16,12 @@
"pathX = \"./train/in.tsv.xz\"\n",
"# pathX = \"./train/in.tsv\"\n",
"pathY = \"./train/expected.tsv\"\n",
"nrows = 10000"
"nrows = 5000"
]
},
{
"cell_type": "code",
"execution_count": 101,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@ -35,15 +35,15 @@
},
{
"cell_type": "code",
"execution_count": 102,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10000\n",
"10000\n"
"5000\n",
"5000\n"
]
}
],
@ -55,7 +55,7 @@
},
{
"cell_type": "code",
"execution_count": 103,
"execution_count": 6,
"metadata": {},
"outputs": [
{
@ -84,8 +84,8 @@
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>8910</th>\n",
" <td>What? It isn't a fake memo. It's a real memo. ...</td>\n",
" <th>2823</th>\n",
" <td>Use her own logic against her. Pharmaceutical...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@ -93,10 +93,10 @@
],
"text/plain": [
" 0\n",
"8910 What? It isn't a fake memo. It's a real memo. ..."
"2823 Use her own logic against her. Pharmaceutical..."
]
},
"execution_count": 103,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@ -107,25 +107,13 @@
},
{
"cell_type": "code",
"execution_count": 104,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array(['00', '000', '00000001', ..., 'αsynuclein', 'ಠ_ಠ', 'fibrosis'],\n",
" dtype=object)"
]
},
"execution_count": 104,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"vectorizer = TfidfVectorizer(lowercase=True, stop_words=['english'])\n",
"X = vectorizer.fit_transform(df.to_numpy().ravel())\n",
"vectorizer.get_feature_names_out()\n"
"# vectorizer.get_feature_names_out()\n"
]
},
{
@ -148,7 +136,7 @@
},
{
"cell_type": "code",
"execution_count": 107,
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
@ -159,17 +147,15 @@
},
{
"cell_type": "code",
"execution_count": 108,
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Python310\\lib\\site-packages\\sklearn\\utils\\validation.py:993: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
" y = column_or_1d(y, warn=True)\n",
"c:\\Python310\\lib\\site-packages\\sklearn\\utils\\validation.py:593: FutureWarning: np.matrix usage is deprecated in 1.0 and will raise a TypeError in 1.2. Please convert to a numpy array with np.asarray. For more information see: https://numpy.org/doc/stable/reference/generated/numpy.matrix.html\n",
" warnings.warn(\n"
"c:\\software\\python3\\lib\\site-packages\\sklearn\\utils\\validation.py:63: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
" return f(*args, **kwargs)\n"
]
},
{
@ -178,7 +164,7 @@
"GaussianNB()"
]
},
"execution_count": 108,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
@ -188,7 +174,7 @@
"from sklearn.naive_bayes import GaussianNB\n",
"\n",
"gnb = GaussianNB()\n",
"gnb.fit(tfidfVector.todense(), topics)"
"gnb.fit(tfidfVector.toarray(), topics)"
]
},
{
@ -208,12 +194,12 @@
},
{
"cell_type": "code",
"execution_count": 115,
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"testXPath = \"./test-A/in.tsv.xz\"\n",
"testYPath = \"./test-A/expected.tsv\"\n",
"testXPath = \"./dev-0/in.tsv.xz\"\n",
"testYPath = \"./dev-0/out.tsv\"\n",
"\n",
"testX = pd.read_csv(testXPath, sep='\\t', nrows=nrows, header=None)\n",
"\n",
@ -223,27 +209,19 @@
},
{
"cell_type": "code",
"execution_count": 118,
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Python310\\lib\\site-packages\\sklearn\\utils\\validation.py:593: FutureWarning: np.matrix usage is deprecated in 1.0 and will raise a TypeError in 1.2. Please convert to a numpy array with np.asarray. For more information see: https://numpy.org/doc/stable/reference/generated/numpy.matrix.html\n",
" warnings.warn(\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0 1 0 ... 0 0 1]\n"
"[0 1 0 ... 1 1 1]\n"
]
}
],
"source": [
"pred = gnb.predict(testXtfidfVector.todense())\n",
"pred = gnb.predict(testXtfidfVector.toarray())\n",
"print(pred)\n",
"\n",
"import csv\n",
@ -255,10 +233,10 @@
],
"metadata": {
"interpreter": {
"hash": "369f2c481f4da34e4445cda3fffd2e751bd1c4d706f27375911949ba6bb62e1c"
"hash": "1b132c2ed43285dcf39f6d01712959169a14a721cf314fe69015adab49bb1fd1"
},
"kernelspec": {
"display_name": "Python 3.10.4 64-bit",
"display_name": "Python 3.8.10 64-bit",
"language": "python",
"name": "python3"
},
@ -272,7 +250,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
"version": "3.8.10"
},
"orig_nbformat": 4
},

1400
test-A/out.tsv
View File

File diff suppressed because it is too large Load Diff