Upgrade clean method

This commit is contained in:
Adrian 2023-04-28 09:55:14 +02:00
parent cf2a6a1363
commit d0d1459d8d

View File

@ -64,8 +64,8 @@
"def clean_text(line: str):\n", "def clean_text(line: str):\n",
" # Preprocessing\n", " # Preprocessing\n",
" separated = line.split('\\t')\n", " separated = line.split('\\t')\n",
" prefix = separated[6].replace(r'\\n', ' ').replace('\\\\n', ' ').replace(' ', ' ').replace('.', '').replace(',', '').replace('?', '').replace('!', '').replace('(', '').replace(')', '').replace(';', '').replace(':', '').replace('\"', '').replace(\"'\", '').replace('-', ' ').replace(' ', ' ').lower()\n", " prefix = separated[6].replace(r'\\n', ' ').replace('\\\\n', ' ').replace(' ', ' ').replace('.', '').replace(',', '').replace('?', '').replace('!', '').replace('(', '').replace(')', '').replace(';', '').replace(':', '').replace('\"', '').replace(\"'\", '').replace('-', ' ').replace(' ', ' ')\n",
" suffix = separated[7].replace(r'\\n', ' ').replace('\\\\n', ' ').replace(' ', ' ').replace('.', '').replace(',', '').replace('?', '').replace('!', '').replace('(', '').replace(')', '').replace(';', '').replace(':', '').replace('\"', '').replace(\"'\", '').replace('-', ' ').replace(' ', ' ').lower()\n", " suffix = separated[7].replace(r'\\n', ' ').replace('\\\\n', ' ').replace(' ', ' ').replace('.', '').replace(',', '').replace('?', '').replace('!', '').replace('(', '').replace(')', '').replace(';', '').replace(':', '').replace('\"', '').replace(\"'\", '').replace('-', ' ').replace(' ', ' ')\n",
" return prefix + ' ' + suffix" " return prefix + ' ' + suffix"
] ]
}, },