s444417/s444417-paranormal-or-skeptic-ISI-public
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
s444417 2022-05-18 10:50:06 +02:00
parent 0fd3ddf4f3
commit 5ad6b81959
3 changed files with 1753 additions and 2199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2472
dev-0/out.tsv
View File

File diff suppressed because it is too large Load Diff

80
run.ipynb
View File

@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 100, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -16,12 +16,12 @@
"pathX = \"./train/in.tsv.xz\"\n", "pathX = \"./train/in.tsv.xz\"\n",
"# pathX = \"./train/in.tsv\"\n", "# pathX = \"./train/in.tsv\"\n",
"pathY = \"./train/expected.tsv\"\n", "pathY = \"./train/expected.tsv\"\n",
"nrows = 10000" "nrows = 5000"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 101, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -35,15 +35,15 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 102, "execution_count": 5,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"10000\n", "5000\n",
"10000\n" "5000\n"
] ]
} }
], ],
@ -55,7 +55,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 103, "execution_count": 6,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -84,8 +84,8 @@
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
" <tr>\n", " <tr>\n",
" <th>8910</th>\n", " <th>2823</th>\n",
" <td>What? It isn't a fake memo. It's a real memo. ...</td>\n", " <td>Use her own logic against her. Pharmaceutical...</td>\n",
" </tr>\n", " </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
@ -93,10 +93,10 @@
], ],
"text/plain": [ "text/plain": [
" 0\n", " 0\n",
"8910 What? It isn't a fake memo. It's a real memo. ..." "2823 Use her own logic against her. Pharmaceutical..."
] ]
}, },
"execution_count": 103, "execution_count": 6,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -107,25 +107,13 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 104, "execution_count": 8,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"data": {
"text/plain": [
"array(['00', '000', '00000001', ..., 'αsynuclein', 'ಠ_ಠ', 'fibrosis'],\n",
" dtype=object)"
]
},
"execution_count": 104,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"vectorizer = TfidfVectorizer(lowercase=True, stop_words=['english'])\n", "vectorizer = TfidfVectorizer(lowercase=True, stop_words=['english'])\n",
"X = vectorizer.fit_transform(df.to_numpy().ravel())\n", "X = vectorizer.fit_transform(df.to_numpy().ravel())\n",
"vectorizer.get_feature_names_out()\n" "# vectorizer.get_feature_names_out()\n"
] ]
}, },
{ {
@ -148,7 +136,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 107, "execution_count": 9,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -159,17 +147,15 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 108, "execution_count": 11,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"c:\\Python310\\lib\\site-packages\\sklearn\\utils\\validation.py:993: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n", "c:\\software\\python3\\lib\\site-packages\\sklearn\\utils\\validation.py:63: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
" y = column_or_1d(y, warn=True)\n", " return f(*args, **kwargs)\n"
"c:\\Python310\\lib\\site-packages\\sklearn\\utils\\validation.py:593: FutureWarning: np.matrix usage is deprecated in 1.0 and will raise a TypeError in 1.2. Please convert to a numpy array with np.asarray. For more information see: https://numpy.org/doc/stable/reference/generated/numpy.matrix.html\n",
" warnings.warn(\n"
] ]
}, },
{ {
@ -178,7 +164,7 @@
"GaussianNB()" "GaussianNB()"
] ]
}, },
"execution_count": 108, "execution_count": 11,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -188,7 +174,7 @@
"from sklearn.naive_bayes import GaussianNB\n", "from sklearn.naive_bayes import GaussianNB\n",
"\n", "\n",
"gnb = GaussianNB()\n", "gnb = GaussianNB()\n",
"gnb.fit(tfidfVector.todense(), topics)" "gnb.fit(tfidfVector.toarray(), topics)"
] ]
}, },
{ {
@ -208,12 +194,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 115, "execution_count": 12,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"testXPath = \"./test-A/in.tsv.xz\"\n", "testXPath = \"./dev-0/in.tsv.xz\"\n",
"testYPath = \"./test-A/expected.tsv\"\n", "testYPath = \"./dev-0/out.tsv\"\n",
"\n", "\n",
"testX = pd.read_csv(testXPath, sep='\\t', nrows=nrows, header=None)\n", "testX = pd.read_csv(testXPath, sep='\\t', nrows=nrows, header=None)\n",
"\n", "\n",
@ -223,27 +209,19 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 118, "execution_count": 13,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Python310\\lib\\site-packages\\sklearn\\utils\\validation.py:593: FutureWarning: np.matrix usage is deprecated in 1.0 and will raise a TypeError in 1.2. Please convert to a numpy array with np.asarray. For more information see: https://numpy.org/doc/stable/reference/generated/numpy.matrix.html\n",
" warnings.warn(\n"
]
},
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"[0 1 0 ... 0 0 1]\n" "[0 1 0 ... 1 1 1]\n"
] ]
} }
], ],
"source": [ "source": [
"pred = gnb.predict(testXtfidfVector.todense())\n", "pred = gnb.predict(testXtfidfVector.toarray())\n",
"print(pred)\n", "print(pred)\n",
"\n", "\n",
"import csv\n", "import csv\n",
@ -255,10 +233,10 @@
], ],
"metadata": { "metadata": {
"interpreter": { "interpreter": {
"hash": "369f2c481f4da34e4445cda3fffd2e751bd1c4d706f27375911949ba6bb62e1c" "hash": "1b132c2ed43285dcf39f6d01712959169a14a721cf314fe69015adab49bb1fd1"
}, },
"kernelspec": { "kernelspec": {
"display_name": "Python 3.10.4 64-bit", "display_name": "Python 3.8.10 64-bit",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@ -272,7 +250,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.10.4" "version": "3.8.10"
}, },
"orig_nbformat": 4 "orig_nbformat": 4
}, },

1400
test-A/out.tsv
View File

File diff suppressed because it is too large Load Diff