118 lines
3.8 KiB
Python
118 lines
3.8 KiB
Python
# Amazon revievs search engine
|
|
from typing import List
|
|
import string
|
|
import pandas as pd
|
|
import numpy as np
|
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
from vectorizer_idf import VectorizerIdf
|
|
from vectorizer_tf import VectorizerTf
|
|
import spacy
|
|
import re
|
|
|
|
|
|
nlp = spacy.load("en_core_web_sm")
|
|
|
|
def get_answers_array():
|
|
d = pd.read_csv('data.csv', engine='python', error_bad_lines=False)
|
|
answers = d["ad"]
|
|
answers = answers.dropna()
|
|
|
|
return np.array(answers)
|
|
|
|
def has_numbers(inputString):
|
|
return any(char.isdigit() for char in inputString)
|
|
|
|
def okapi_mb25(query, tf, idf, a_len, documents, d_idf):
|
|
k = 1.6
|
|
b = 0.75
|
|
scores = []
|
|
q_tf = tf.get_tf_for_document(query)
|
|
docs_v = tf.tf_matrix.toarray()
|
|
for index, document in enumerate(documents):
|
|
s = 0
|
|
for re_p in re.findall('[0-9a-z.,-/]+', document):
|
|
for q_ in query.split():
|
|
if re_p == q_:
|
|
s +=100
|
|
for word in document.split():
|
|
val_tf = 0
|
|
for d in query.split():
|
|
if d == word:
|
|
val_tf += 1
|
|
if not d_idf.get(word):
|
|
continue
|
|
idf_for_word = d_idf[word]
|
|
licznik = val_tf * (k + 1)
|
|
mianownik = val_tf + k * (1 - b + b * (len(document.split()) / a_len))
|
|
s += idf_for_word * (licznik / mianownik)
|
|
scores.append(s)
|
|
return scores
|
|
# try:
|
|
# # v_tf = VectorizerTf([document])
|
|
# # tf_for_doc = v_tf.get_tf_for_document(query)
|
|
# # tf_for_document = v_tf.tf_matrix.toarray() * tf_for_doc[0]
|
|
# doc_v = docs_v[index]
|
|
# a=1
|
|
# for idx, val in enumerate(doc_v):
|
|
# pass
|
|
# # idf_for_word = idf.matrix.toarray()[index][idx]
|
|
# # licznik = val * (k + 1)
|
|
# # mianownik = val + k * (1 - b + b * (len(document.split()) / a_len))
|
|
# # s += idf_for_word * (licznik / mianownik)
|
|
# # a=1
|
|
# # scores.append(s)
|
|
# except Exception as e:
|
|
# scores.append(0)
|
|
# print('error', e)
|
|
# return scores
|
|
|
|
|
|
def preprocess(d_list: List):
|
|
result = []
|
|
for d in d_list:
|
|
words = []
|
|
d = d.translate(str.maketrans('', '', string.punctuation))
|
|
for token in nlp(d):
|
|
words.append(token.lemma_)
|
|
result.append(" ".join(words))
|
|
return result
|
|
|
|
if __name__ == "__main__":
|
|
data = get_answers_array()
|
|
# data = ['Ala has a cat', 'Maciej and Ala have dog and Ala Ala Ala', 'Ala has a turtle', 'Maciej has a dog, turtle and a lot of cats',
|
|
# 'Ola has a dog, turtle and cat, butters']
|
|
data = [d.lower() for d in data]
|
|
data = np.array(data)
|
|
# data = preprocess(data)
|
|
|
|
|
|
average_lens = []
|
|
for doc in data:
|
|
words = doc.split()
|
|
if words:
|
|
average_lens.append(sum(len(word) for word in words) / len(words))
|
|
average_doc_len = sum(average_lens) / len(average_lens)
|
|
|
|
vectorizer_tf = VectorizerTf(data)
|
|
vectorizer_idf = VectorizerIdf(data)
|
|
vocab = vectorizer_tf.feature_names
|
|
data_idf = {}
|
|
for idx, v in enumerate(vocab):
|
|
data_idf[v] = vectorizer_idf.get_idf_for_word(v)
|
|
|
|
while True:
|
|
q = input('Wpisz fraze: ')
|
|
q = q.lower()
|
|
# q = preprocess([q])[0]
|
|
score = okapi_mb25(q, vectorizer_tf, vectorizer_idf, average_doc_len, data, data_idf)
|
|
print('loading ended')
|
|
list1, list2 = zip(*sorted(zip(score, data)))
|
|
i = 0
|
|
for sc, sent in zip(reversed(list1), reversed(list2)):
|
|
if sc:
|
|
print(sent, sc)
|
|
i += 1
|
|
if i == 5:
|
|
break
|
|
X = [i for i in score if i != 0]
|
|
print('Znaleziono ' + str(len(X)) + ' wyniki') |