ium_464913/dataset-stats.py

import os
import sys
import pandas as pd


def write_to_file(file_name):
    df = pd.read_csv("data/creditcard.csv")
    pd.set_option("display.max_columns", None)

    X_train = pd.read_csv("data/X_train.csv")
    X_val = pd.read_csv("data/X_val.csv")
    X_test = pd.read_csv("data/X_test.csv")
    y_train = pd.read_csv("data/y_train.csv")
    y_val = pd.read_csv("data/y_val.csv")
    y_test = pd.read_csv("data/y_test.csv")

    with open("stats_data/" + file_name, "w") as f:
        sys.stdout = f

        f.write("Check missing values\n")
        f.write(str(df.isnull().sum()))

        f.write("\n\n")

        f.write("Size of the dataset\n")
        df.info()

        f.write("\n\n")

        f.write("Summary statistics\n")
        f.write(str(df.describe()))

        f.write("\n\n")

        f.write("Distribution of legitimate and fraudulent transactions\n")
        f.write(str(df["Class"].value_counts()))

        f.write("\n\n")

        f.write("Statistical measures of the training dataset\n")
        pd.concat([X_train, y_train], axis=1).info()
        f.write("\n")
        f.write(str(pd.concat([X_train, y_train], axis=1).describe()))
        f.write("\n")
        f.write(str(pd.concat([X_train, y_train], axis=1)["Class"].value_counts()))

        f.write("\n\n")

        f.write("Statistical measures of the validation dataset\n")
        pd.concat([X_val, y_val], axis=1).info()
        f.write("\n")
        f.write(str(pd.concat([X_val, y_val], axis=1).describe()))
        f.write("\n")
        f.write(str(pd.concat([X_val, y_val], axis=1)["Class"].value_counts()))

        f.write("\n\n")

        f.write("Statistical measures of the test dataset\n")
        pd.concat([X_test, y_test], axis=1).info()
        f.write("\n")
        f.write(str(pd.concat([X_test, y_test], axis=1).describe()))
        f.write("\n")
        f.write(str(pd.concat([X_test, y_test], axis=1)["Class"].value_counts()))

        sys.stdout = sys.__stdout__


def main():
    os.makedirs("stats_data", exist_ok=True)
    os.system("rm -rf stats_data/*")
    write_to_file("stats.txt")


if __name__ == "__main__":
    main()