add IUM-05

This commit is contained in:
jakubknczny 2021-04-24 13:31:02 +02:00
parent c1207ceda5
commit 0ce98b3ed7
8 changed files with 812 additions and 9 deletions

View File

@ -12,4 +12,12 @@ and prints a short summary of the dataset as well as its subsets.
### Zadanie 2
add Jenkinsfiles and mock data preprocessing
### Zadanie 5
added lab4 file with new python script and updated Dockerfile.
The container downloads the dataset and installs software needed,
then trains and evaluates model on the dataset.
Loss and accuracy are saved to test_eval.txt file.
ium01.ipynb is a notebook used to develop previously mentioned scripts.

File diff suppressed because one or more lines are too long

View File

@ -14,7 +14,9 @@ pipeline {
}
stage('docker') {
agent {
dockerfile true
docker {
image 'kubakonieczny/ium:v1.0'
}
}
stages {
stage('script') {

View File

@ -1,11 +1,37 @@
node {
docker.image('kubakonieczny/ium:v1.0').withRun("-t -e KAGGLE_USERNAME=kubakonieczny -e KAGGLE_KEY=${params.KAGGLE_KEY}") { c ->
docker.image('kubakonieczny/ium:v1.0').inside {
stage('Test') {
sh 'cat /etc/issue'
sh 'ls -lah'
pipeline {
agent none
stages {
stage('copy files') {
agent any
steps {
sh '''
cp ./lab3/script.sh .
cp ./lab3/python_script.py .
cp ./lab3/Dockerfile .
cp ./lab3/requirements.txt .
'''
}
}
stage('docker') {
agent {
docker {
image 'kubakonieczny/ium:v1.0'
}
}
stages {
stage('script') {
steps {
sh '''
chmod +x script.sh
./script.sh > stats.txt'''
}
}
stage('archive artifact') {
steps {
archiveArtifacts 'stats.txt'
}
}
}
}
}
sh 'docker logs ${c.id}'
}

19
lab4/Dockerfile Normal file
View File

@ -0,0 +1,19 @@
FROM ubuntu:latest
RUN apt update >>/dev/null
RUN apt install -y apt-utils >>/dev/null
RUN apt install -y python3.8 >>/dev/null
RUN apt install -y python3-pip >>/dev/null
RUN apt install -y unzip >>/dev/null
WORKDIR /app
COPY ./test_eval.py ./
COPY ./script.sh ./
RUN chmod +x script.sh
COPY ./requirements.txt ./
RUN pip3 install -r requirements.txt >>/dev/null
CMD ./script.sh

5
lab4/requirements.txt Normal file
View File

@ -0,0 +1,5 @@
kaggle
numpy~=1.19.2
pandas
sklearn
tensorflow

6
lab4/script.sh Normal file
View File

@ -0,0 +1,6 @@
#!/bin/bash
kaggle datasets download -d 'pcbreviglieri/smart-grid-stability'
unzip smart-grid-stability.zip >>/dev/null 2>&1
python3 test_eval.py

61
lab4/test_eval.py Normal file
View File

@ -0,0 +1,61 @@
import numpy as np
import pandas as pd
import tensorflow as tf
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from tensorflow.keras import layers
def onezero(label):
return 0 if label == 'unstable' else 1
df = pd.read_csv('smart_grid_stability_augmented.csv')
scaler = preprocessing.StandardScaler().fit(df.iloc[:, 0:-1])
df_norm_array = scaler.transform(df.iloc[:, 0:-1])
df_norm = pd.DataFrame(data=df_norm_array,
columns=df.columns[:-1])
df_norm['stabf'] = df['stabf']
df_norm_data = df_norm.copy()
df_norm_data = df_norm_data.drop('stab', axis=1)
df_norm_labels = df_norm_data.pop('stabf')
X_train, X_testAndValid, Y_train, Y_testAndValid = train_test_split(
df_norm_data,
df_norm_labels,
test_size=0.2,
random_state=42)
X_test, X_valid, Y_test, Y_valid = train_test_split(
X_testAndValid,
Y_testAndValid,
test_size=0.5,
random_state=42)
model = tf.keras.Sequential([
layers.Input(shape=(12,)),
layers.Dense(32),
layers.Dense(16),
layers.Dense(2, activation='softmax')
])
model.compile(
loss=tf.losses.BinaryCrossentropy(),
optimizer=tf.optimizers.Adam(),
metrics=[tf.keras.metrics.BinaryAccuracy()])
Y_train_one_zero = [onezero(x) for x in Y_train]
Y_train_onehot = np.eye(2)[Y_train_one_zero]
Y_test_one_zero = [onezero(x) for x in Y_test]
Y_test_onehot = np.eye(2)[Y_test_one_zero]
history = model.fit(tf.convert_to_tensor(X_train, np.float32), Y_train_onehot, epochs=5)
results = model.evaluate(X_test, Y_test_onehot, batch_size=64)
f = open('model_eval.txt', 'w')
f.write('test loss: ' + str(results[0]) + '\n' + 'test acc: ' + str(results[1]))
f.close()