ium_z486867/create-dataset.py

import os

from sklearn.model_selection import train_test_split
import pandas as pd

pd.set_option('display.max_columns', 100)


DATA_DIRECTORY = './ium_z434686'

CSV_NAME = DATA_DIRECTORY + '/openpowerlifting.csv'

def process_data(csv_name):
    CUTOFF = int(os.environ['CUTOFF'])

    powerlifting_data = pd.read_csv(csv_name,
                                  engine='python',
                                  encoding='ISO-8859-1',
                                  sep=',')
    powerlifting_data.dropna()
    powerlifting_data.drop(columns=["Squat4Kg", "Bench4Kg", "Deadlift4Kg"], inplace=True)

    powerlifting_data.sample(CUTOFF)

    X, Y = powerlifting_data, powerlifting_data

    X_train, X_temp, Y_train, Y_temp = train_test_split(X, Y, test_size=0.3, random_state=1)
    X_dev, X_test, Y_dev, Y_test = train_test_split(X_temp, Y_temp, test_size=0.3, random_state=1)

    X_train.to_csv('X_train.csv', index=False)
    X_dev.to_csv('X_dev.csv', index=False)
    X_test.to_csv('X_test.csv', index=False)

process_data(CSV_NAME)