ium_464979/dataset_stats.py

22 lines
659 B
Python
Raw Permalink Normal View History

2024-04-02 22:12:47 +02:00
import os
import subprocess
train_file = "data/train.csv"
dev_file = "data/dev.csv"
test_file = "data/test.csv"
def process_file(file_path, output_file):
result = subprocess.run(['wc', '-l', file_path], capture_output=True, text=True)
with open(output_file, 'w') as f:
f.write(result.stdout)
process_file(train_file, "data/stats_train.txt")
process_file(dev_file, "data/stats_dev.txt")
process_file(test_file, "data/stats_test.txt")
os.makedirs("data", exist_ok=True)
os.rename("data/stats_train.txt", "data/stats_train.txt")
os.rename("data/stats_dev.txt", "data/stats_dev.txt")
os.rename("data/stats_test.txt", "data/stats_test.txt")