uczenie_glebokie/word2vec/word2vec_3.py
2024-05-20 00:10:24 +02:00

83 lines
3.2 KiB
Python

import os
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.utils import class_weight
from gensim.models import KeyedVectors
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.optimizers import Adam
import sys
# Ustawienie kodowania na utf-8
sys.stdout.reconfigure(encoding='utf-8')
# Funkcja do ładowania danych
def load_and_prepare_data(file_path, text_column='text', label_column='label', is_test=False):
if is_test:
df = pd.read_csv(file_path, sep='\t', header=None, names=[text_column], on_bad_lines='skip')
df['label'] = 0 # Dodanie kolumny label z wartością domyślną
else:
df = pd.read_csv(file_path, sep='\t', header=None, names=[label_column, text_column], on_bad_lines='skip')
df[text_column].fillna('', inplace=True)
return df
# 1. Przygotowanie danych
train_df = load_and_prepare_data('./train/train.tsv')
dev_df = load_and_prepare_data('./dev-0/in.tsv', is_test=True)
test_df = load_and_prepare_data('./test-A/in.tsv', is_test=True)
dev_labels = pd.read_csv('./dev-0/expected.tsv', sep='\t', header=None, names=['label'])
# 2. Przygotowanie modelu word2vec
word2vec_path = './word2vec_100_3_polish.bin'
w2v_model = KeyedVectors.load(word2vec_path)
# 3. Przekształcenie tekstów na wektory
def text_to_vector(text, model_w2v, vector_size=100):
if not isinstance(text, str):
return np.zeros((vector_size,))
words = text.split()
words = [word for word in words if word in model_w2v]
if words:
return np.mean(model_w2v[words], axis=0)
else:
return np.zeros((vector_size,))
# 4. Budowa i trenowanie modelu
def build_and_train_model(X_train, y_train):
model = Sequential([
Dense(256, activation='relu', input_dim=100),
Dropout(0.3),
Dense(128, activation='relu'),
Dropout(0.3),
Dense(64, activation='relu'),
Dropout(0.3),
Dense(32, activation='relu'),
Dropout(0.3),
Dense(1, activation='sigmoid')
])
model.compile(optimizer=Adam(learning_rate=0.0005), loss='binary_crossentropy', metrics=['accuracy'])
weights = class_weight.compute_class_weight('balanced', classes=np.unique(y_train), y=y_train)
weights_dict = dict(enumerate(weights))
model.fit(X_train, y_train, epochs=50, batch_size=32, class_weight=weights_dict, validation_split=0.2)
return model
# 5. Generowanie wyników
def generate_results(df, model, filepath, text_column='text'):
X_test = np.array([text_to_vector(text, w2v_model) for text in df[text_column]])
predictions = (model.predict(X_test) > 0.5).astype(int)
pd.DataFrame(predictions).to_csv(filepath, sep='\t', index=False, header=False)
# Przekształcenie tekstów na wektory
X_train = np.array([text_to_vector(text, w2v_model) for text in train_df['text']])
y_train = train_df['label'].values
# Trenowanie modelu
model = build_and_train_model(X_train, y_train)
# Generowanie wyników dla zbioru walidacyjnego i testowego
generate_results(dev_df, model, "./dev-0/out.tsv")
generate_results(test_df, model, "./test-A/out.tsv")