ium_z434686/create-dataset.py

import pandas
import os

from keras.applications.densenet import layers

from sklearn.model_selection import train_test_split
import tensorflow


CUTOFF = int(os.environ['CUTOFF'])

# READ DATA
video_games = pandas.read_csv('./ium_z434686/Video_Games_Sales_as_at_22_Dec_2016.csv',
                              engine='python',
                              encoding='ISO-8859-1',
                              sep=',')

# DROP NA FIELDS
video_games = video_games.dropna()
video_games = video_games.drop(video_games.columns[[0, 1, 3, 4, 13, 14, 15]], axis=1)

# CUT OFF DATASET TO X LINES
video_games = video_games.sample(CUTOFF)

X = video_games.copy()
Y = pandas.DataFrame(video_games.pop('User_Score'))

# SPLIT BETWEEN DEV, TRAINS, AND TEST
X_train, X_temp, Y_train, Y_temp = train_test_split(X, Y, test_size=0.3, random_state=1)
X_dev, X_test, Y_dev, Y_test = train_test_split(X_temp, Y_temp, test_size=0.3, random_state=1)

X_train.to_csv('X_train.csv', index=False)
X_dev.to_csv('X_dev.csv', index=False)
X_test.to_csv('X_test.csv', index=False)
Y_test.to_csv('Y_test.csv', index=False)
Y_train.to_csv('Y_train.csv', index=False)
Y_dev.to_csv('Y_dev.csv', index=False)