s464905/sport-text-classification-ball
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
3e3a3f17c6
sport-text-classification-ball/Word2Vec.ipynb

18 KiB
Raw Blame History

Word2Vec

Import bibliotek

from gensim.models import KeyedVectors
from gensim.utils import simple_preprocess
import pandas as pd
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from sklearn.preprocessing import LabelEncoder

Wczytanie danych

data_train = pd.read_csv('train/train.tsv', sep="\t", names=["Text"], usecols=[1])
data_test = pd.read_csv('test-A/in.tsv', sep="\t", names=["Text"])
data_dev = pd.read_csv('dev-0/in.tsv', sep="\t", names=["Text"])

labels_train = pd.read_csv('train/train.tsv', sep='\t', header=None, names=['Label'], usecols=[0])
labels_dev = pd.read_csv('dev-0/expected.tsv', sep='\t', header=None, names=['Label'])

display(data_train.head())
display(data_test.head())
display(data_dev.head())
display(labels_train.head())
display(labels_dev.head())
Text
0 Mindaugas Budzinauskas wierzy w odbudowę formy...
1 Przyjmujący reprezentacji Polski wrócił do PGE...
2 FEN 9: Zapowiedź walki Róża Gumienna vs Katarz...
3 Aleksander Filipiak: Czuję się dobrze w nowym ...
4 Victoria Carl i Aleksiej Czerwotkin mistrzami ...
Text
0 ATP Sztokholm: Juergen Zopp wykorzystał szansę...
1 Krowicki z reprezentacją kobiet aż do igrzysk ...
2 Wielki powrót Łukasza Kubota Odradza się zawsz...
3 Marcel Hirscher wygrał ostatni slalom gigant m...
4 Polki do Czarnogóry z pełnią zaangażowania. Sy...
Text
0 Mundial 2018. Były reprezentant Anglii trenere...
1 Liga Mistrzyń: Podopieczne Kima Rasmussena bli...
2 Wyczerpujące treningi biegowe Justyny Kowalczy...
3 Mundial 2018. Zagraniczne media zareagowały na...
4 BCL. Artur Gronek: Musimy grać twardziej. Pope...
Label
0 1
1 1
2 0
3 1
4 0
Label
0 1
1 1
2 0
3 1
4 1

Załadowanie wektorów Word2Vec

W2V_model = KeyedVectors.load('fasttext_100_3_polish.bin')

Funkcja przekształcania tekstu na wektory

def text_to_vector(text, word2vec, vector_size):
    words = simple_preprocess(text)
    text_vector = np.zeros(vector_size)
    word_count = 0
    for word in words:
        if word in word2vec.wv:
            text_vector += word2vec.wv[word]
            word_count += 1
    if word_count > 0:
        text_vector /= word_count
    return text_vector

Dostosowanie formatu danych do modelu

# Zamiana tekstów na wektory
train_vectors = np.array([text_to_vector(text, W2V_model, 100) for text in data_train['Text']])
dev_vectors = np.array([text_to_vector(text, W2V_model, 100) for text in data_dev['Text']])
test_vectors = np.array([text_to_vector(text, W2V_model, 100) for text in data_test['Text']])

# Zamiana etykiet na liczby
label_encoder = LabelEncoder()
train_labels_enc = label_encoder.fit_transform(labels_train['Label'])
dev_labels_enc = label_encoder.transform(labels_dev['Label'])

Stworzenie modelu

# Stworzenie modelu
model = Sequential()
model.add(Dense(128, input_dim=100, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

# Trening modelu
model.fit(train_vectors, train_labels_enc, epochs=10, batch_size=32, validation_data=(dev_vectors, dev_labels_enc))
C:\Users\nkaro\AppData\Roaming\Python\Python311\site-packages\keras\src\layers\core\dense.py:86: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/10
3067/3067 ━━━━━━━━━━━━━━━━━━━━ 4s 897us/step - accuracy: 0.9072 - loss: 0.2176 - val_accuracy: 0.9563 - val_loss: 0.1158
Epoch 2/10
3067/3067 ━━━━━━━━━━━━━━━━━━━━ 3s 847us/step - accuracy: 0.9524 - loss: 0.1215 - val_accuracy: 0.9574 - val_loss: 0.1047
Epoch 3/10
3067/3067 ━━━━━━━━━━━━━━━━━━━━ 3s 858us/step - accuracy: 0.9581 - loss: 0.1080 - val_accuracy: 0.9618 - val_loss: 0.0956
Epoch 4/10
3067/3067 ━━━━━━━━━━━━━━━━━━━━ 3s 839us/step - accuracy: 0.9610 - loss: 0.1008 - val_accuracy: 0.9648 - val_loss: 0.0949
Epoch 5/10
3067/3067 ━━━━━━━━━━━━━━━━━━━━ 3s 850us/step - accuracy: 0.9640 - loss: 0.0951 - val_accuracy: 0.9547 - val_loss: 0.1071
Epoch 6/10
3067/3067 ━━━━━━━━━━━━━━━━━━━━ 3s 872us/step - accuracy: 0.9643 - loss: 0.0928 - val_accuracy: 0.9631 - val_loss: 0.0913
Epoch 7/10
3067/3067 ━━━━━━━━━━━━━━━━━━━━ 3s 845us/step - accuracy: 0.9654 - loss: 0.0886 - val_accuracy: 0.9659 - val_loss: 0.0911
Epoch 8/10
3067/3067 ━━━━━━━━━━━━━━━━━━━━ 3s 841us/step - accuracy: 0.9669 - loss: 0.0860 - val_accuracy: 0.9642 - val_loss: 0.0889
Epoch 9/10
3067/3067 ━━━━━━━━━━━━━━━━━━━━ 3s 853us/step - accuracy: 0.9666 - loss: 0.0856 - val_accuracy: 0.9642 - val_loss: 0.0855
Epoch 10/10
3067/3067 ━━━━━━━━━━━━━━━━━━━━ 3s 841us/step - accuracy: 0.9676 - loss: 0.0821 - val_accuracy: 0.9666 - val_loss: 0.0883

<keras.src.callbacks.history.History at 0x1d1185b4b50>

Predykcja i zapis danych wyjścowych

# Predykcje dla danych walidacyjnych
dev_predictions = model.predict(dev_vectors)
dev_predictions = (dev_predictions > 0.5).astype(int)

# Predykcje dla danych testowych
test_predictions = model.predict(test_vectors)
test_predictions = (test_predictions > 0.5).astype(int)

# Zapisanie wyników do plików
pd.DataFrame(dev_predictions).to_csv('dev-0/out.tsv', sep='\t', index=False, header=False)
pd.DataFrame(test_predictions).to_csv('test-A/out.tsv', sep='\t', index=False, header=False)
171/171 ━━━━━━━━━━━━━━━━━━━━ 1s 4ms/step
171/171 ━━━━━━━━━━━━━━━━━━━━ 0s 638us/step