challenging-america-word-ga.../kenlm.ipynb

1 line
65 KiB
Plaintext
Raw Normal View History

2023-05-10 00:37:23 +02:00
{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyOjlR0HzrxQLi9ivvf3rrhL"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"kgUXhu_9HEZY","executionInfo":{"status":"ok","timestamp":1682427020888,"user_tz":-120,"elapsed":7836,"user":{"displayName":"Martyna Drumińska","userId":"13361003509289187965"}},"outputId":"93b9b737-532d-4892-d4bf-66579ee7c849"},"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","source":["cd drive/MyDrive"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4gxwwa5-Haqo","executionInfo":{"status":"ok","timestamp":1682427020889,"user_tz":-120,"elapsed":13,"user":{"displayName":"Martyna Drumińska","userId":"13361003509289187965"}},"outputId":"d5a1b591-43f9-4810-fb49-bf247c1a08e2"},"execution_count":7,"outputs":[{"output_type":"stream","name":"stdout","text":["[Errno 2] No such file or directory: 'drive/MyDrive'\n","/content/drive/MyDrive/challenging-america-word-gap-prediction\n"]}]},{"cell_type":"code","source":["cd challenging-america-word-gap-prediction/"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"78igYakJHokM","executionInfo":{"status":"ok","timestamp":1682427020891,"user_tz":-120,"elapsed":12,"user":{"displayName":"Martyna Drumińska","userId":"13361003509289187965"}},"outputId":"c1906f10-600a-4170-b61f-ab3005a2cf2a"},"execution_count":8,"outputs":[{"output_type":"stream","name":"stdout","text":["[Errno 2] No such file or directory: 'challenging-america-word-gap-prediction/'\n","/content/drive/MyDrive/challenging-america-word-gap-prediction\n"]}]},{"cell_type":"code","source":["import pandas as pd"],"metadata":{"id":"-wyIUdlBHp2W","executionInfo":{"status":"ok","timestamp":1682427020892,"user_tz":-120,"elapsed":9,"user":{"displayName":"Martyna Drumińska","userId":"13361003509289187965"}}},"execution_count":9,"outputs":[]},{"cell_type":"code","source":["data = pd.read_csv(\"train/in.tsv.xz\", sep=\"\\t\", on_bad_lines='skip', header=None, encoding=\"utf-8\")\n","\n","exp_words = pd.read_csv(\"train/expected.tsv\", sep=\"\\t\", on_bad_lines='skip', header=None, encoding=\"utf-8\")"],"metadata":{"id":"kA6PExReHr3E","executionInfo":{"status":"ok","timestamp":1682430631336,"user_tz":-120,"elapsed":39975,"user":{"displayName":"Martyna Drumińska","userId":"13361003509289187965"}}},"execution_count":28,"outputs":[]},{"cell_type":"code","execution_count":29,"metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1682430631338,"user":{"displayName":"Martyna Drumińska","userId":"13361003509289187965"},"user_tz":-120},"id":"upTQ5Po9wOSL"},"outputs":[],"source":["train_data = data[[6, 7]]"]},{"cell_type":"code","execution_count":30,"metadata":{"executionInfo":{"elapsed":18,"status":"ok","timestamp":1682430631341,"user":{"displayName":"Martyna Drumińska","userId":"13361003509289187965"},"user_tz":-120},"id":"OdEm_SBSwXuY"},"outputs":[],"source":["train_data= pd.concat([train_data, exp_words], axis=1)"]},{"cell_type":"code","execution_count":31,"metadata":{"executionInfo":{"elapsed":19,"status":"ok","timestamp":1682430631343,"user":{"displayName":"Martyna Drumińska","userId":"13361003509289187965"},"user_tz":-120},"id":"b1TM741wwYdA"},"outputs":[],"source":["train_data.rename(columns={6: 'First Part', 7: 'Second Part', 0:'Expected word'}, inplace=True)"]},{"cell_type":"code","execution_count":32,"metadata":{"executionInfo":{"elapsed":1675,"status":"ok","timestamp":1682430633001,"user":{"displayName":"Martyna Drumińska","userId":"13361003509289187965"},"user_tz":-120},"id":"6Zfbmp-IxKUH"},"outputs":[],"source":["train_data['Concatenated'] = train_data['First Part'] + train_data['Expected word'] + train_data['Second