12 lines
189 B
Python
12 lines
189 B
Python
|
#!/usr/bin/python3
|
||
|
|
|
||
|
|
from nltk.tokenize import word_tokenize
|
||
|
|
import nltk
|
||
|
|
import re
|
||
|
|
import string
|
||
|
|
|
||
|
|
def tokenize(d):
|
||
|
|
d = re.sub(r'\\n',' ',d)
|
||
|
|
words = word_tokenize(d)
|
||
|
|
return words
|