7 lines
146 B
Python
7 lines
146 B
Python
|
#!/usr/bin/env python3
|
||
|
|
import nltk
|
||
|
|
nltk.download('punkt')
|
||
|
|
from nltk.tokenize import word_tokenize
|
||
|
|
def tokenizator(d):
|
||
|
|
return word_tokenize(d)
|