add scrapper for comments, endpoint for scrapping
This commit is contained in:
parent
e4786e2027
commit
081b161ca2
@ -1,6 +1,7 @@
|
||||
from transformers import AutoTokenizer
|
||||
from transformers import pipeline
|
||||
import re
|
||||
from facebook_scraper import get_posts
|
||||
|
||||
model = 'application/models/sentiment_model'
|
||||
tokenizer = AutoTokenizer.from_pretrained('application/tokenizers/sentiment_tokenizer')
|
||||
@ -35,4 +36,11 @@ def count_predictions(predictions):
|
||||
all['negative'] = l0
|
||||
all['neutral'] = l2
|
||||
|
||||
return all
|
||||
return all
|
||||
|
||||
def scrapp_comments(url):
|
||||
result = []
|
||||
for post in get_posts(post_urls=[url], options={"allow_extra_requests": False, "comments":True, "extra_info":True}):
|
||||
for comment in post['comments_full']:
|
||||
result.append(comment['comment_text'])
|
||||
return result
|
||||
|
@ -3,7 +3,7 @@ from flask import(
|
||||
jsonify,
|
||||
Blueprint,
|
||||
)
|
||||
from application.functions.sentiment import sentiment_prediction, count_predictions, clear_data
|
||||
from application.functions.sentiment import sentiment_prediction, count_predictions, clear_data, scrapp_comments
|
||||
|
||||
sentiment_service = Blueprint("sentiment_service", __name__)
|
||||
|
||||
@ -18,4 +18,11 @@ def get_data():
|
||||
predicitons[i]['sentence'] = data['sentences'][i]
|
||||
predicitons[i]['label'] = predicitons[i]['label'][6:]
|
||||
|
||||
return jsonify({"predictions": predicitons, "count_labels": count_labels})
|
||||
return jsonify({"predictions": predicitons, "count_labels": count_labels})
|
||||
|
||||
|
||||
@sentiment_service.route("/scrapp_comments", methods=['POST'])
|
||||
def scrapp():
|
||||
url = request.get_json()
|
||||
comments = scrapp_comments(url['link'])
|
||||
return jsonify({'comments':comments})
|
Loading…
Reference in New Issue
Block a user