62 lines
1.6 KiB
Python
62 lines
1.6 KiB
Python
|
# %%
|
|||
|
|
import pandas as pd
|
|||
|
|
import os
|
|||
|
|
import re
|
|||
|
|
# %% [markdown]
|
|||
|
|
### Reading data - this part need changing when data
|
|||
|
|
# %%
|
|||
|
|
path = os.getcwd()
|
|||
|
|
filename = 'training_data_clean.csv'
|
|||
|
|
filepath = path+'/'+filename
|
|||
|
|
data = pd.read_csv(filepath, header=None,
|
|||
|
|
delimiter=',', encoding_errors='surrogateescape')
|
|||
|
|
data.columns = ['index', 'id','date', 'query', 'user', 'text']
|
|||
|
|
# %% [markdown]
|
|||
|
|
### Function definitions
|
|||
|
|
# %%
|
|||
|
|
change_dict = {
|
|||
|
|
# tokens
|
|||
|
|
"USERNAME": ['@\w+|@'],
|
|||
|
|
"URL": ['http\S*'],
|
|||
|
|
"EMOJI": ["[;:][dbop\(\)\[\]]|[dbop\(\)\[\]][;:]|xd+|\S*&\S*"],
|
|||
|
|
# standardization
|
|||
|
|
', ': ['\s,'],
|
|||
|
|
'. ': ['\s\.'],
|
|||
|
|
' ': ['\s{2,}'],
|
|||
|
|
"'": ["<EFBFBD>"],
|
|||
|
|
'?': ["\s\?+|\?+"],
|
|||
|
|
'!': ["\s\!+|\!+"]
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def clean_lines(line, change_dict):
|
|||
|
|
line = line.lower()
|
|||
|
|
for change_to, change_regex_list in change_dict.items():
|
|||
|
|
for change_regex in change_regex_list:
|
|||
|
|
line = re.sub(change_regex, change_to, line)
|
|||
|
|
return line
|
|||
|
|
|
|||
|
|
def get_rep_idx_to_cut_out_from_str(line):
|
|||
|
|
occurence = 0
|
|||
|
|
idx_to_cut = []
|
|||
|
|
for idx, letter in enumerate(line):
|
|||
|
|
if idx > 0:
|
|||
|
|
occurence = occurence+1 if line[idx-1] == letter else 0
|
|||
|
|
if occurence >= 2:
|
|||
|
|
idx_to_cut.append(idx)
|
|||
|
|
return idx_to_cut
|
|||
|
|
|
|||
|
|
def truncate_duplicated_letters_to_two(line):
|
|||
|
|
idx_to_cut = get_rep_idx_to_cut_out_from_str(line)
|
|||
|
|
str_out =''
|
|||
|
|
for i,s in enumerate(line):
|
|||
|
|
if i not in idx_to_cut:
|
|||
|
|
str_out += s
|
|||
|
|
return str_out
|
|||
|
|
# %% [markdown]
|
|||
|
|
### Cleaning
|
|||
|
|
# %%
|
|||
|
|
text = [clean_lines(x, change_dict) for x in data.loc[:, 'text'].values.tolist()]
|
|||
|
|
text = [truncate_duplicated_letters_to_two(x).strip() for x in text]
|
|||
|
|
data.text = text
|
|||
|
|
# %%
|