65 lines
1.5 KiB
Python
65 lines
1.5 KiB
Python
import string
|
|
import re
|
|
import nltk
|
|
from nltk.tokenize import word_tokenize
|
|
from nltk.corpus import stopwords
|
|
from nltk.stem import WordNetLemmatizer
|
|
|
|
nltk.download('stopwords')
|
|
nltk.download('wordnet')
|
|
nltk.download('punkt')
|
|
|
|
|
|
# remove urls, handles, and the hashtag from hashtags (taken from https://stackoverflow.com/questions/8376691/how-to-remove-hashtag-user-link-of-a-tweet-using-regular-expression)
|
|
def remove_urls(text):
|
|
new_text = ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",text).split())
|
|
return new_text
|
|
|
|
|
|
# make all text lowercase
|
|
def text_lowercase(text):
|
|
return text.lower()
|
|
|
|
|
|
# remove numbers
|
|
def remove_numbers(text):
|
|
result = re.sub(r'\d+', '', text)
|
|
return result
|
|
|
|
|
|
# remove punctuation
|
|
def remove_punctuation(text):
|
|
translator = str.maketrans('', '', string.punctuation)
|
|
return text.translate(translator)
|
|
|
|
|
|
# tokenize
|
|
def tokenize(text):
|
|
text = word_tokenize(text)
|
|
return text
|
|
|
|
|
|
# remove stopwords
|
|
stop_words = set(stopwords.words('english'))
|
|
def remove_stopwords(text):
|
|
text = [i for i in text if not i in stop_words]
|
|
return text
|
|
|
|
# lemmatize
|
|
lemmatizer = WordNetLemmatizer()
|
|
def lemmatize(text):
|
|
text = [lemmatizer.lemmatize(token) for token in text]
|
|
return text
|
|
|
|
|
|
def preprocess_text(text):
|
|
text = text_lowercase(text)
|
|
text = remove_urls(text)
|
|
text = remove_numbers(text)
|
|
text = remove_punctuation(text)
|
|
text = tokenize(text)
|
|
text = remove_stopwords(text)
|
|
text = lemmatize(text)
|
|
text = ' '.join(text)
|
|
return text
|