224 lines
13 KiB
Plaintext
224 lines
13 KiB
Plaintext
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 1,
|
||
|
|
"id": "equal-singles",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/utils/validation.py:37: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n",
|
||
|
|
" LARGE_SPARSE_SUPPORTED = LooseVersion(scipy_version) >= '0.14.0'\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/feature_extraction/image.py:167: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" dtype=np.int):\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/linear_model/least_angle.py:35: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" eps=np.finfo(np.float).eps,\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/linear_model/least_angle.py:597: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" eps=np.finfo(np.float).eps, copy_X=True, fit_path=True,\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/linear_model/least_angle.py:836: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" eps=np.finfo(np.float).eps, copy_X=True, fit_path=True,\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/linear_model/least_angle.py:862: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" eps=np.finfo(np.float).eps, positive=False):\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/linear_model/least_angle.py:1097: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" max_n_alphas=1000, n_jobs=None, eps=np.finfo(np.float).eps,\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/linear_model/least_angle.py:1344: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" max_n_alphas=1000, n_jobs=None, eps=np.finfo(np.float).eps,\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/linear_model/least_angle.py:1480: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" eps=np.finfo(np.float).eps, copy_X=True, positive=False):\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/linear_model/randomized_l1.py:152: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" precompute=False, eps=np.finfo(np.float).eps,\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/linear_model/randomized_l1.py:320: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" eps=np.finfo(np.float).eps, random_state=None,\n",
|
||
|
|
"/usr/lib/python3/dist-packages/sklearn/linear_model/randomized_l1.py:580: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
|
||
|
|
"Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
|
||
|
|
" eps=4 * np.finfo(np.float).eps, n_jobs=None,\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"import numpy as np\n",
|
||
|
|
"import pandas as pd\n",
|
||
|
|
"import torch\n",
|
||
|
|
"import csv\n",
|
||
|
|
"import lzma\n",
|
||
|
|
"import gensim.downloader\n",
|
||
|
|
"from nltk import word_tokenize"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 2,
|
||
|
|
"id": "involved-understanding",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"x_train = pd.read_table('in.tsv', sep='\\t', header=None, quoting=3)\n",
|
||
|
|
"y_train = pd.read_table('expected.tsv', sep='\\t', header=None, quoting=3)\n",
|
||
|
|
"#x_dev = pd.read_table('dev-0/in.tsv.xz', compression='xz', sep='\\t', header=None, quoting=3)\n",
|
||
|
|
"#x_test = pd.read_table('test-A/in.tsv.xz', compression='xz', sep='\\t', header=None, quoting=3)\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 5,
|
||
|
|
"id": "collaborative-cincinnati",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"ename": "AttributeError",
|
||
|
|
"evalue": "module 'torch' has no attribute 'nn'",
|
||
|
|
"output_type": "error",
|
||
|
|
"traceback": [
|
||
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||
|
|
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||
|
|
"\u001b[0;32m<ipython-input-5-11c9482004ae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m#print('inicjalizacja modelu')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mNeuralNetworkModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mNeuralNetworkModel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ml01\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m300\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||
|
|
"\u001b[0;31mAttributeError\u001b[0m: module 'torch' has no attribute 'nn'"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"#print('inicjalizacja modelu')\n",
|
||
|
|
"class NeuralNetworkModel(torch.nn.Module):\n",
|
||
|
|
" def __init__(self):\n",
|
||
|
|
" super(NeuralNetworkModel, self).__init__()\n",
|
||
|
|
" self.l01 = torch.nn.Linear(300, 300)\n",
|
||
|
|
" self.l02 = torch.nn.Linear(300, 1)\n",
|
||
|
|
"\n",
|
||
|
|
" def forward(self, x):\n",
|
||
|
|
" x = self.l01(x)\n",
|
||
|
|
" x = torch.relu(x)\n",
|
||
|
|
" x = self.l02(x)\n",
|
||
|
|
" x = torch.sigmoid(x)\n",
|
||
|
|
" return x"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "hydraulic-business",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"#print('przygotowanie danych')\n",
|
||
|
|
"\n",
|
||
|
|
"x_train = x_train[0].str.lower()\n",
|
||
|
|
"y_train = y_train[0]\n",
|
||
|
|
"x_dev = x_dev[0].str.lower()\n",
|
||
|
|
"x_test = x_test[0].str.lower()\n",
|
||
|
|
"\n",
|
||
|
|
"x_train = [word_tokenize(x) for x in x_train]\n",
|
||
|
|
"x_dev = [word_tokenize(x) for x in x_dev]\n",
|
||
|
|
"x_test = [word_tokenize(x) for x in x_test]\n",
|
||
|
|
"\n",
|
||
|
|
"word2vec = gensim.downloader.load('word2vec-google-news-300')\n",
|
||
|
|
"x_train = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_train]\n",
|
||
|
|
"x_dev = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_dev]\n",
|
||
|
|
"x_test = [np.mean([word2vec[word] for word in content if word in word2vec] or [np.zeros(300)], axis=0) for content in x_test]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "heavy-sandwich",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"#print('trenowanie modelu')\n",
|
||
|
|
"model = NeuralNetworkModel()\n",
|
||
|
|
"BATCH_SIZE = 5\n",
|
||
|
|
"criterion = torch.nn.BCELoss()\n",
|
||
|
|
"optimizer = torch.optim.SGD(model.parameters(), lr=0.01)\n",
|
||
|
|
"\n",
|
||
|
|
"for epoch in range(BATCH_SIZE):\n",
|
||
|
|
" model.train()\n",
|
||
|
|
" for i in range(0, y_train.shape[0], BATCH_SIZE):\n",
|
||
|
|
" X = x_train[i:i + BATCH_SIZE]\n",
|
||
|
|
" X = torch.tensor(X)\n",
|
||
|
|
" y = y_train[i:i + BATCH_SIZE]\n",
|
||
|
|
" y = torch.tensor(y.astype(np.float32).to_numpy()).reshape(-1, 1)\n",
|
||
|
|
" optimizer.zero_grad()\n",
|
||
|
|
" outputs = model(X.float())\n",
|
||
|
|
" loss = criterion(outputs, y)\n",
|
||
|
|
" loss.backward()\n",
|
||
|
|
" optimizer.step()"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "small-pavilion",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"#print('predykcja wynikow')\n",
|
||
|
|
"y_dev = []\n",
|
||
|
|
"y_test = []\n",
|
||
|
|
"model.eval()\n",
|
||
|
|
"\n",
|
||
|
|
"with torch.no_grad():\n",
|
||
|
|
" for i in range(0, len(x_dev), BATCH_SIZE):\n",
|
||
|
|
" X = x_dev[i:i + BATCH_SIZE]\n",
|
||
|
|
" X = torch.tensor(X)\n",
|
||
|
|
" outputs = model(X.float())\n",
|
||
|
|
" prediction = (outputs > 0.5)\n",
|
||
|
|
" y_dev += prediction.tolist()\n",
|
||
|
|
"\n",
|
||
|
|
" for i in range(0, len(x_test), BATCH_SIZE):\n",
|
||
|
|
" X = x_test[i:i + BATCH_SIZE]\n",
|
||
|
|
" X = torch.tensor(X)\n",
|
||
|
|
" outputs = model(X.float())\n",
|
||
|
|
" y = (outputs >= 0.5)\n",
|
||
|
|
" y_test += prediction.tolist()\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "toxic-pendant",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"# print('eksportowanie do plików')\n",
|
||
|
|
"y_dev = np.asarray(y_dev, dtype=np.int32)\n",
|
||
|
|
"y_test = np.asarray(y_test, dtype=np.int32)\n",
|
||
|
|
"y_dev.tofile('./dev-0/out.tsv', sep='\\n')\n",
|
||
|
|
"y_test.tofile('./test-A/out.tsv', sep='\\n')\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.7.3"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 5
|
||
|
|
}
|