7 lines
146 B
Python
Executable File
7 lines
146 B
Python
Executable File
#!/usr/bin/env python3
|
|
import nltk
|
|
nltk.download('punkt')
|
|
from nltk.tokenize import word_tokenize
|
|
def tokenizator(d):
|
|
return word_tokenize(d)
|