feat: sacred
This commit is contained in:
Jakub Zaręba
parent
b2d430825e
commit
cf89b66ba2
92
evaluate.py
92
evaluate.py
@ -1,44 +1,68 @@
|
|||||||
import tensorflow as tf
|
from sacred import Experiment
|
||||||
import pandas as pd
|
from sacred.observers import MongoObserver, FileStorageObserver
|
||||||
import numpy as np
|
|
||||||
from sklearn.preprocessing import MinMaxScaler
|
|
||||||
from sklearn.metrics import accuracy_score, f1_score, mean_squared_error
|
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
import os
|
|
||||||
|
|
||||||
model = tf.keras.models.load_model('model.h5')
|
ex = Experiment('s487187_experiment', interactive=True)
|
||||||
|
ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017', db_name='sacred'))
|
||||||
|
ex.observers.append(FileStorageObserver('results'))
|
||||||
|
|
||||||
test_data = pd.read_csv('data.csv', sep=';')
|
@ex.config
|
||||||
test_data = pd.get_dummies(test_data, columns=['Sex', 'Medal'])
|
def my_config():
|
||||||
test_data = test_data.drop(columns=['Name', 'Team', 'NOC', 'Games', 'Year', 'Season', 'City', 'Sport', 'Event'])
|
model_path = 'model.h5'
|
||||||
|
test_data_path = 'data.csv'
|
||||||
|
metrics_file_path = 'metrics.txt'
|
||||||
|
plot_path = 'plot.png'
|
||||||
|
|
||||||
scaler = MinMaxScaler()
|
@ex.capture
|
||||||
test_data = pd.DataFrame(scaler.fit_transform(test_data), columns=test_data.columns)
|
def evaluate_model(model_path, test_data_path, metrics_file_path, plot_path):
|
||||||
|
import tensorflow as tf
|
||||||
|
import pandas as pd
|
||||||
|
import numpy as np
|
||||||
|
from sklearn.preprocessing import MinMaxScaler
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
import os
|
||||||
|
|
||||||
X_test = test_data.filter(regex='Sex|Age')
|
model = tf.keras.models.load_model(model_path)
|
||||||
y_test = test_data.filter(regex='Medal')
|
|
||||||
y_test = pd.get_dummies(y_test)
|
|
||||||
|
|
||||||
X_test = X_test.fillna(0)
|
test_data = pd.read_csv(test_data_path, sep=';')
|
||||||
y_test = y_test.fillna(0)
|
test_data = pd.get_dummies(test_data, columns=['Sex', 'Medal'])
|
||||||
|
test_data = test_data.drop(columns=['Name', 'Team', 'NOC', 'Games', 'Year', 'Season', 'City', 'Sport', 'Event'])
|
||||||
|
|
||||||
y_pred = model.predict(X_test)
|
scaler = MinMaxScaler()
|
||||||
|
test_data = pd.DataFrame(scaler.fit_transform(test_data), columns=test_data.columns)
|
||||||
|
|
||||||
top_1_accuracy = tf.keras.metrics.categorical_accuracy(y_test, y_pred)
|
X_test = test_data.filter(regex='Sex|Age')
|
||||||
top_5_accuracy = tf.keras.metrics.top_k_categorical_accuracy(y_test, y_pred, k=5)
|
y_test = test_data.filter(regex='Medal')
|
||||||
|
y_test = pd.get_dummies(y_test)
|
||||||
|
|
||||||
metrics_file = 'metrics.txt'
|
X_test = X_test.fillna(0)
|
||||||
if os.path.exists(metrics_file):
|
y_test = y_test.fillna(0)
|
||||||
metrics_df = pd.read_csv(metrics_file)
|
|
||||||
else:
|
|
||||||
metrics_df = pd.DataFrame(columns=['top_1_accuracy', 'top_5_accuracy'])
|
|
||||||
|
|
||||||
new_row = pd.DataFrame([{'top_1_accuracy': np.mean(top_1_accuracy.numpy()), 'top_5_accuracy': np.mean(top_5_accuracy.numpy())}])
|
y_pred = model.predict(X_test)
|
||||||
metrics_df = pd.concat([metrics_df, new_row], ignore_index=True)
|
|
||||||
metrics_df.to_csv(metrics_file, index=False)
|
|
||||||
|
|
||||||
plt.figure(figsize=(10, 6))
|
top_1_accuracy = tf.keras.metrics.categorical_accuracy(y_test, y_pred)
|
||||||
plt.plot(metrics_df['top_1_accuracy'], label='Top-1 Accuracy')
|
top_5_accuracy = tf.keras.metrics.top_k_categorical_accuracy(y_test, y_pred, k=5)
|
||||||
plt.plot(metrics_df['top_5_accuracy'], label='Top-5 Accuracy')
|
|
||||||
plt.legend()
|
if os.path.exists(metrics_file_path):
|
||||||
plt.savefig('plot.png')
|
metrics_df = pd.read_csv(metrics_file_path)
|
||||||
|
else:
|
||||||
|
metrics_df = pd.DataFrame(columns=['top_1_accuracy', 'top_5_accuracy'])
|
||||||
|
|
||||||
|
new_row = pd.DataFrame([{'top_1_accuracy': np.mean(top_1_accuracy.numpy()), 'top_5_accuracy': np.mean(top_5_accuracy.numpy())}])
|
||||||
|
metrics_df = pd.concat([metrics_df, new_row], ignore_index=True)
|
||||||
|
metrics_df.to_csv(metrics_file_path, index=False)
|
||||||
|
|
||||||
|
plt.figure(figsize=(10, 6))
|
||||||
|
plt.plot(metrics_df['top_1_accuracy'], label='Top-1 Accuracy')
|
||||||
|
plt.plot(metrics_df['top_5_accuracy'], label='Top-5 Accuracy')
|
||||||
|
plt.legend()
|
||||||
|
plt.savefig(plot_path)
|
||||||
|
|
||||||
|
ex.log_scalar('top_1_accuracy', np.mean(top_1_accuracy.numpy()))
|
||||||
|
ex.log_scalar('top_5_accuracy', np.mean(top_5_accuracy.numpy()))
|
||||||
|
ex.add_artifact(model_path)
|
||||||
|
ex.add_artifact(metrics_file_path)
|
||||||
|
ex.add_artifact(plot_path)
|
||||||
|
|
||||||
|
@ex.automain
|
||||||
|
def main():
|
||||||
|
evaluate_model()
|
||||||
|
|||||||
82
train.py
82
train.py
@ -1,46 +1,68 @@
|
|||||||
import pandas as pd
|
from sacred import Experiment
|
||||||
from sklearn.model_selection import train_test_split
|
from sacred.observers import MongoObserver, FileStorageObserver
|
||||||
from sklearn.preprocessing import MinMaxScaler
|
|
||||||
import tensorflow as tf
|
|
||||||
from imblearn.over_sampling import SMOTE
|
|
||||||
|
|
||||||
smote = SMOTE(random_state=42)
|
ex = Experiment('s487187-training')
|
||||||
data = pd.read_csv('data.csv', sep=';')
|
ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017', db_name='sacred'))
|
||||||
|
ex.observers.append(FileStorageObserver('results'))
|
||||||
|
|
||||||
print('Total rows:', len(data))
|
@ex.config
|
||||||
print('Rows with medal:', len(data.dropna(subset=['Medal'])))
|
def my_config():
|
||||||
|
data_file = 'data.csv'
|
||||||
|
model_file = 'model.h5'
|
||||||
|
epochs = 10
|
||||||
|
batch_size = 32
|
||||||
|
test_size = 0.2
|
||||||
|
random_state = 42
|
||||||
|
|
||||||
|
@ex.capture
|
||||||
|
def train_model(data_file, model_file, epochs, batch_size, test_size, random_state):
|
||||||
|
import pandas as pd
|
||||||
|
from sklearn.model_selection import train_test_split
|
||||||
|
from sklearn.preprocessing import MinMaxScaler
|
||||||
|
import tensorflow as tf
|
||||||
|
from imblearn.over_sampling import SMOTE
|
||||||
|
|
||||||
data = pd.get_dummies(data, columns=['Sex', 'Medal'])
|
smote = SMOTE(random_state=random_state)
|
||||||
|
data = pd.read_csv(data_file, sep=';')
|
||||||
|
|
||||||
data = data.drop(columns=['Name', 'Team', 'NOC', 'Games', 'Year', 'Season', 'City', 'Sport', 'Event'])
|
print('Total rows:', len(data))
|
||||||
|
print('Rows with medal:', len(data.dropna(subset=['Medal'])))
|
||||||
|
|
||||||
scaler = MinMaxScaler()
|
data = pd.get_dummies(data, columns=['Sex', 'Medal'])
|
||||||
data = pd.DataFrame(scaler.fit_transform(data), columns=data.columns)
|
data = data.drop(columns=['Name', 'Team', 'NOC', 'Games', 'Year', 'Season', 'City', 'Sport', 'Event'])
|
||||||
|
|
||||||
X = data.filter(regex='Sex|Age')
|
scaler = MinMaxScaler()
|
||||||
y = data.filter(regex='Medal')
|
data = pd.DataFrame(scaler.fit_transform(data), columns=data.columns)
|
||||||
y = pd.get_dummies(y)
|
|
||||||
|
|
||||||
X = X.fillna(0)
|
X = data.filter(regex='Sex|Age')
|
||||||
y = y.fillna(0)
|
y = data.filter(regex='Medal')
|
||||||
|
y = pd.get_dummies(y)
|
||||||
|
|
||||||
y = y.values
|
X = X.fillna(0)
|
||||||
|
y = y.fillna(0)
|
||||||
|
|
||||||
X_resampled, y_resampled = smote.fit_resample(X, y)
|
y = y.values
|
||||||
X_train, X_test, y_train, y_test = train_test_split(X_resampled, y_resampled, test_size=0.2, random_state=42)
|
|
||||||
|
|
||||||
model = tf.keras.models.Sequential()
|
X_resampled, y_resampled = smote.fit_resample(X, y)
|
||||||
model.add(tf.keras.layers.Dense(64, input_dim=X_train.shape[1], activation='relu'))
|
X_train, X_test, y_train, y_test = train_test_split(X_resampled, y_resampled, test_size=test_size, random_state=random_state)
|
||||||
model.add(tf.keras.layers.Dense(32, activation='relu'))
|
|
||||||
model.add(tf.keras.layers.Dense(y.shape[1], activation='softmax'))
|
|
||||||
|
|
||||||
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
|
model = tf.keras.models.Sequential()
|
||||||
|
model.add(tf.keras.layers.Dense(64, input_dim=X_train.shape[1], activation='relu'))
|
||||||
|
model.add(tf.keras.layers.Dense(32, activation='relu'))
|
||||||
|
model.add(tf.keras.layers.Dense(y.shape[1], activation='softmax'))
|
||||||
|
|
||||||
model.fit(X_train, y_train, epochs=10, batch_size=32)
|
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
|
||||||
loss, accuracy = model.evaluate(X_test, y_test)
|
|
||||||
print('Test accuracy:', accuracy)
|
|
||||||
|
|
||||||
model.save('model.h5')
|
model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size)
|
||||||
|
loss, accuracy = model.evaluate(X_test, y_test)
|
||||||
|
print('Test accuracy:', accuracy)
|
||||||
|
|
||||||
|
model.save(model_file)
|
||||||
|
|
||||||
|
return accuracy
|
||||||
|
|
||||||
|
@ex.main
|
||||||
|
def run_experiment():
|
||||||
|
accuracy = train_model()
|
||||||
|
ex.log_scalar('accuracy', accuracy)
|
||||||
|
ex.add_artifact('model.h5')
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user