s464914/word2vec
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Upload files to "/"

Browse Source
This commit is contained in:
s464914 2024-05-19 18:27:03 +02:00
commit 90cb1368c5
4 changed files with 109450 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5447
in.tsv Normal file
View File

File diff suppressed because it is too large Load Diff

5445
out.tsv Normal file
View File

File diff suppressed because it is too large Load Diff

98132
train.tsv Normal file
View File

File diff suppressed because it is too large Load Diff

426
word2vec.ipynb Normal file
View File

@ -0,0 +1,426 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-19T15:41:42.652253200Z",
"start_time": "2024-05-19T15:41:39.981058Z"
}
},
"outputs": [],
"source": [
"import gensim\n",
"import numpy as np\n",
"import pandas as pd\n",
"from sklearn.model_selection import train_test_split\n",
"pd.set_option('display.max_colwidth', 100)"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-19T16:06:17.776931600Z",
"start_time": "2024-05-19T16:06:16.000064500Z"
}
},
"outputs": [
{
"data": {
"text/plain": " 1 \\\n0 1 \n1 0 \n2 1 \n3 0 \n4 1 \n\n Mindaugas Budzinauskas wierzy w odbudowę formy Kevina Johnsona. Czy ktoś opuści Polpharmę? Mindaugas Budzinauskas w rozmowie z WP SportoweFakty opowiada o transferze Kevina Johnsona, ewentualnych odejściach z Polpharmy i kolejnym meczu PLK z Anwilem. - Potrzebowaliśmy takiego gracza, jak Johnson - podkreśla szkoleniowiec starogardzian. \n0 Przyjmujący reprezentacji Polski wrócił do PGE Skry Bełchatów Tylko rok trwał rozbrat Artura Sza... \n1 FEN 9: Zapowiedź walki Róża Gumienna vs Katarzyna Posiadała (wideo) Podczas Fight Exclusive Nigh... \n2 Aleksander Filipiak: Czuję się dobrze w nowym klubie Aleksander Filipiak w przerwie letniej zami... \n3 Victoria Carl i Aleksiej Czerwotkin mistrzami świata juniorów na 5 i 10 kilometrów Biegi na 5 i ... \n4 Świat poznał ją na mundialu. Francuska WAG czaruje pięknym ciałem Rachel Legrain-Trapani to jedn... ",
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>1</th>\n <th>Mindaugas Budzinauskas wierzy w odbudowę formy Kevina Johnsona. Czy ktoś opuści Polpharmę? Mindaugas Budzinauskas w rozmowie z WP SportoweFakty opowiada o transferze Kevina Johnsona, ewentualnych odejściach z Polpharmy i kolejnym meczu PLK z Anwilem. - Potrzebowaliśmy takiego gracza, jak Johnson - podkreśla szkoleniowiec starogardzian.</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>1</td>\n <td>Przyjmujący reprezentacji Polski wrócił do PGE Skry Bełchatów Tylko rok trwał rozbrat Artura Sza...</td>\n </tr>\n <tr>\n <th>1</th>\n <td>0</td>\n <td>FEN 9: Zapowiedź walki Róża Gumienna vs Katarzyna Posiadała (wideo) Podczas Fight Exclusive Nigh...</td>\n </tr>\n <tr>\n <th>2</th>\n <td>1</td>\n <td>Aleksander Filipiak: Czuję się dobrze w nowym klubie Aleksander Filipiak w przerwie letniej zami...</td>\n </tr>\n <tr>\n <th>3</th>\n <td>0</td>\n <td>Victoria Carl i Aleksiej Czerwotkin mistrzami świata juniorów na 5 i 10 kilometrów Biegi na 5 i ...</td>\n </tr>\n <tr>\n <th>4</th>\n <td>1</td>\n <td>Świat poznał ją na mundialu. Francuska WAG czaruje pięknym ciałem Rachel Legrain-Trapani to jedn...</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data = pd.read_csv(\"train.tsv\", sep=\"\\t\", on_bad_lines='skip')\n",
"data.head(5)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-19T16:06:19.207507600Z",
"start_time": "2024-05-19T16:06:19.145007800Z"
}
},
"outputs": [
{
"data": {
"text/plain": " label \\\n0 1 \n1 0 \n2 1 \n3 0 \n4 1 \n\n text \n0 Przyjmujący reprezentacji Polski wrócił do PGE Skry Bełchatów Tylko rok trwał rozbrat Artura Sza... \n1 FEN 9: Zapowiedź walki Róża Gumienna vs Katarzyna Posiadała (wideo) Podczas Fight Exclusive Nigh... \n2 Aleksander Filipiak: Czuję się dobrze w nowym klubie Aleksander Filipiak w przerwie letniej zami... \n3 Victoria Carl i Aleksiej Czerwotkin mistrzami świata juniorów na 5 i 10 kilometrów Biegi na 5 i ... \n4 Świat poznał ją na mundialu. Francuska WAG czaruje pięknym ciałem Rachel Legrain-Trapani to jedn... ",
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>label</th>\n <th>text</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>1</td>\n <td>Przyjmujący reprezentacji Polski wrócił do PGE Skry Bełchatów Tylko rok trwał rozbrat Artura Sza...</td>\n </tr>\n <tr>\n <th>1</th>\n <td>0</td>\n <td>FEN 9: Zapowiedź walki Róża Gumienna vs Katarzyna Posiadała (wideo) Podczas Fight Exclusive Nigh...</td>\n </tr>\n <tr>\n <th>2</th>\n <td>1</td>\n <td>Aleksander Filipiak: Czuję się dobrze w nowym klubie Aleksander Filipiak w przerwie letniej zami...</td>\n </tr>\n <tr>\n <th>3</th>\n <td>0</td>\n <td>Victoria Carl i Aleksiej Czerwotkin mistrzami świata juniorów na 5 i 10 kilometrów Biegi na 5 i ...</td>\n </tr>\n <tr>\n <th>4</th>\n <td>1</td>\n <td>Świat poznał ją na mundialu. Francuska WAG czaruje pięknym ciałem Rachel Legrain-Trapani to jedn...</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.columns = [\"label\", \"text\"]\n",
"data.head(5)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-19T16:06:33.740107200Z",
"start_time": "2024-05-19T16:06:24.643641200Z"
}
},
"outputs": [
{
"data": {
"text/plain": " label \\\n0 1 \n1 0 \n2 1 \n3 0 \n4 1 \n\n text \\\n0 Przyjmujący reprezentacji Polski wrócił do PGE Skry Bełchatów Tylko rok trwał rozbrat Artura Sza... \n1 FEN 9: Zapowiedź walki Róża Gumienna vs Katarzyna Posiadała (wideo) Podczas Fight Exclusive Nigh... \n2 Aleksander Filipiak: Czuję się dobrze w nowym klubie Aleksander Filipiak w przerwie letniej zami... \n3 Victoria Carl i Aleksiej Czerwotkin mistrzami świata juniorów na 5 i 10 kilometrów Biegi na 5 i ... \n4 Świat poznał ją na mundialu. Francuska WAG czaruje pięknym ciałem Rachel Legrain-Trapani to jedn... \n\n text_preprocessed \n0 [przyjmujący, reprezentacji, polski, wrócił, do, pge, skry, bełchatów, tylko, rok, trwał, rozbra... \n1 [fen, zapowiedź, walki, róża, gumienna, vs, katarzyna, posiadała, wideo, podczas, fight, exclusi... \n2 [aleksander, filipiak, czuję, się, dobrze, nowym, klubie, aleksander, filipiak, przerwie, letnie... \n3 [victoria, carl, aleksiej, czerwotkin, mistrzami, świata, juniorów, na, kilometrów, biegi, na, k... \n4 [świat, poznał, ją, na, mundialu, francuska, wag, czaruje, pięknym, ciałem, rachel, legrain, tra... ",
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>label</th>\n <th>text</th>\n <th>text_preprocessed</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>1</td>\n <td>Przyjmujący reprezentacji Polski wrócił do PGE Skry Bełchatów Tylko rok trwał rozbrat Artura Sza...</td>\n <td>[przyjmujący, reprezentacji, polski, wrócił, do, pge, skry, bełchatów, tylko, rok, trwał, rozbra...</td>\n </tr>\n <tr>\n <th>1</th>\n <td>0</td>\n <td>FEN 9: Zapowiedź walki Róża Gumienna vs Katarzyna Posiadała (wideo) Podczas Fight Exclusive Nigh...</td>\n <td>[fen, zapowiedź, walki, róża, gumienna, vs, katarzyna, posiadała, wideo, podczas, fight, exclusi...</td>\n </tr>\n <tr>\n <th>2</th>\n <td>1</td>\n <td>Aleksander Filipiak: Czuję się dobrze w nowym klubie Aleksander Filipiak w przerwie letniej zami...</td>\n <td>[aleksander, filipiak, czuję, się, dobrze, nowym, klubie, aleksander, filipiak, przerwie, letnie...</td>\n </tr>\n <tr>\n <th>3</th>\n <td>0</td>\n <td>Victoria Carl i Aleksiej Czerwotkin mistrzami świata juniorów na 5 i 10 kilometrów Biegi na 5 i ...</td>\n <td>[victoria, carl, aleksiej, czerwotkin, mistrzami, świata, juniorów, na, kilometrów, biegi, na, k...</td>\n </tr>\n <tr>\n <th>4</th>\n <td>1</td>\n <td>Świat poznał ją na mundialu. Francuska WAG czaruje pięknym ciałem Rachel Legrain-Trapani to jedn...</td>\n <td>[świat, poznał, ją, na, mundialu, francuska, wag, czaruje, pięknym, ciałem, rachel, legrain, tra...</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data['text_preprocessed'] = data['text'].apply(lambda x: gensim.utils.simple_preprocess(x))\n",
"data.head(5)"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-19T16:06:55.614995200Z",
"start_time": "2024-05-19T16:06:38.098671700Z"
}
},
"outputs": [],
"source": [
"from gensim.test.utils import common_texts\n",
"from gensim.models import Word2Vec\n",
"\n",
"model = Word2Vec(sentences=data['text_preprocessed'], vector_size=100, window=5, min_count=1, workers=4)"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-19T16:07:14.652019200Z",
"start_time": "2024-05-19T16:07:06.839062700Z"
}
},
"outputs": [],
"source": [
"import tensorflow as tf\n",
"\n",
"token = tf.keras.preprocessing.text.Tokenizer(1234)\n",
"token.fit_on_texts(data['text_preprocessed'])\n",
"text = token.texts_to_sequences(data['text_preprocessed'])\n",
"text = tf.keras.utils.pad_sequences(text, 75)"
]
},
{
"cell_type": "code",
"execution_count": 41,
"outputs": [
{
"data": {
"text/plain": "array([[ 0, 0, 0, ..., 189, 416, 346],\n [ 0, 0, 0, ..., 56, 397, 907],\n [ 0, 0, 0, ..., 137, 226, 343],\n ...,\n [ 0, 0, 0, ..., 817, 3, 317],\n [ 0, 0, 0, ..., 656, 20, 923],\n [ 0, 0, 0, ..., 6, 478, 27]])"
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"X_train, X_test, y_train, y_test = train_test_split(np.array(text), data['label'], test_size=0.3)\n",
"X_train"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
},
"ExecuteTime": {
"end_time": "2024-05-19T16:07:16.936539700Z",
"start_time": "2024-05-19T16:07:16.829393500Z"
}
}
},
{
"cell_type": "code",
"execution_count": 42,
"outputs": [],
"source": [
"vocab_size = len(token.word_index) + 1\n",
"embedding_matrix = np.zeros((vocab_size, 100))\n",
"\n",
"for word, i in token.word_index.items():\n",
" if word in model.wv:\n",
" embedding_matrix[i] = model.wv[word]\n",
" \n",
"import keras\n",
"\n"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-05-19T16:07:20.210193100Z",
"start_time": "2024-05-19T16:07:19.694625200Z"
}
}
},
{
"cell_type": "code",
"execution_count": 48,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/10\n",
"1074/1074 [==============================] - 45s 41ms/step - loss: 0.1928 - acc: 0.9313 - val_loss: 0.1255 - val_acc: 0.9564\n",
"Epoch 2/10\n",
"1074/1074 [==============================] - 43s 40ms/step - loss: 0.1985 - acc: 0.9309 - val_loss: 0.1173 - val_acc: 0.9543\n",
"Epoch 3/10\n",
"1074/1074 [==============================] - 44s 41ms/step - loss: 0.2170 - acc: 0.9252 - val_loss: 0.1254 - val_acc: 0.9522\n",
"Epoch 4/10\n",
"1074/1074 [==============================] - 44s 41ms/step - loss: 0.1953 - acc: 0.9291 - val_loss: 0.1191 - val_acc: 0.9524\n",
"Epoch 5/10\n",
"1074/1074 [==============================] - 44s 41ms/step - loss: 0.2065 - acc: 0.9253 - val_loss: 0.1237 - val_acc: 0.9503\n",
"Epoch 6/10\n",
"1074/1074 [==============================] - 43s 40ms/step - loss: 0.1990 - acc: 0.9275 - val_loss: 0.1334 - val_acc: 0.9452\n",
"Epoch 7/10\n",
"1074/1074 [==============================] - 44s 41ms/step - loss: 0.1960 - acc: 0.9256 - val_loss: 0.1266 - val_acc: 0.9516\n",
"Epoch 8/10\n",
"1074/1074 [==============================] - 45s 42ms/step - loss: 0.2203 - acc: 0.9217 - val_loss: 0.1327 - val_acc: 0.9458\n",
"Epoch 9/10\n",
"1074/1074 [==============================] - 43s 40ms/step - loss: 0.2149 - acc: 0.9197 - val_loss: 0.1191 - val_acc: 0.9486\n",
"Epoch 10/10\n",
"1074/1074 [==============================] - 69s 64ms/step - loss: 0.2136 - acc: 0.9232 - val_loss: 0.1187 - val_acc: 0.9511\n"
]
},
{
"data": {
"text/plain": "<keras.src.callbacks.History at 0x26e8784f9d0>"
},
"execution_count": 48,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from keras.src.layers import Dropout, Dense, Activation, Embedding, MaxPooling1D, GlobalMaxPooling1D\n",
"from keras.src.layers import Conv1D\n",
"from keras import Sequential\n",
"\n",
"opt = keras.optimizers.Adam(learning_rate=0.01)\n",
"\n",
"keras_model = Sequential()\n",
"keras_model.add(Embedding(vocab_size, 100, weights=[embedding_matrix], trainable=False))\n",
"keras_model.add(Dropout(0.4))\n",
"keras_model.add(Conv1D(100, 3, activation='relu', padding='same', strides=1))\n",
"keras_model.add(Conv1D(100, 3, activation='relu', padding='same', strides=1))\n",
"keras_model.add(GlobalMaxPooling1D())\n",
"keras_model.add(Dropout(0.4))\n",
"keras_model.add(Dense(200))\n",
"keras_model.add(Activation('relu'))\n",
"keras_model.add(Dropout(0.4))\n",
"keras_model.add(Dense(1))\n",
"keras_model.add(Activation('sigmoid'))\n",
"keras_model.compile(loss='binary_crossentropy', metrics=['acc'], optimizer=opt)\n",
"keras_model.fit(X_train, y_train, batch_size=64, epochs=10, validation_data=(X_test, y_test))"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-05-19T16:20:23.872101800Z",
"start_time": "2024-05-19T16:12:39.852425600Z"
}
}
},
{
"cell_type": "code",
"execution_count": 49,
"outputs": [],
"source": [
"model = keras_model"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-05-19T16:23:52.063773Z",
"start_time": "2024-05-19T16:23:52.048148Z"
}
}
},
{
"cell_type": "code",
"execution_count": 50,
"outputs": [],
"source": [
"def preprocess(path):\n",
" data = pd.read_csv(path, sep=\"\\t\", on_bad_lines='skip')\n",
" data.columns = [\"text\"]\n",
" data['text_clean'] = data['text'].apply(lambda x: gensim.utils.simple_preprocess(x))\n",
" text = token.texts_to_sequences(data['text_clean'])\n",
" text = tf.keras.utils.pad_sequences(text, 75)\n",
" return text"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-05-19T16:23:59.273586100Z",
"start_time": "2024-05-19T16:23:59.226688700Z"
}
}
},
{
"cell_type": "code",
"execution_count": 51,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"171/171 [==============================] - 2s 8ms/step\n"
]
}
],
"source": [
"x = preprocess(\"in.tsv\")\n",
"res = model.predict(x)\n",
"y_predictions = np.where(res >= 0.49, 1, 0)\n",
"out = pd.DataFrame(y_predictions)"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-05-19T16:24:04.905985Z",
"start_time": "2024-05-19T16:24:01.955988500Z"
}
}
},
{
"cell_type": "code",
"execution_count": 53,
"outputs": [],
"source": [
"y_predictions = np.where(res>=0.49, 1, 0) "
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-05-19T16:24:25.889023300Z",
"start_time": "2024-05-19T16:24:25.842145800Z"
}
}
},
{
"cell_type": "code",
"execution_count": 54,
"outputs": [],
"source": [
"out = pd.DataFrame(y_predictions)"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-05-19T16:24:31.504270200Z",
"start_time": "2024-05-19T16:24:31.410519700Z"
}
}
},
{
"cell_type": "code",
"execution_count": 55,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" 0\n",
"0 1\n",
"1 1\n",
"2 0\n",
"3 1\n",
"4 1\n",
"... ..\n",
"5439 1\n",
"5440 1\n",
"5441 1\n",
"5442 0\n",
"5443 1\n",
"\n",
"[5444 rows x 1 columns]\n"
]
}
],
"source": [
"print(out)"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-05-19T16:24:37.105738100Z",
"start_time": "2024-05-19T16:24:37.027601500Z"
}
}
},
{
"cell_type": "code",
"execution_count": 56,
"outputs": [],
"source": [
"out.to_csv('out.tsv', sep=\"\\t\", index=False) "
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-05-19T16:24:46.318087Z",
"start_time": "2024-05-19T16:24:46.239966300Z"
}
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [],
"metadata": {
"collapsed": false
}
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}