add scrapper for comments, endpoint for scrapping

application/functions/sentiment.py

@@ -1,6 +1,7 @@
 from transformers import AutoTokenizer
 from transformers import pipeline
 import re
+from facebook_scraper import get_posts
 
 model = 'application/models/sentiment_model'
 tokenizer = AutoTokenizer.from_pretrained('application/tokenizers/sentiment_tokenizer')
@@ -35,4 +36,11 @@ def count_predictions(predictions):
     all['negative'] = l0
     all['neutral'] = l2
 
-    return all
+    return all
+
+def scrapp_comments(url):
+    result = []
+    for post in get_posts(post_urls=[url], options={"allow_extra_requests": False, "comments":True, "extra_info":True}):
+        for comment in post['comments_full']:
+            result.append(comment['comment_text'])
+    return result

application/services/sentiment_service.py

@@ -3,7 +3,7 @@ from flask import(
     jsonify, 
     Blueprint,
     )
-from application.functions.sentiment import sentiment_prediction, count_predictions, clear_data
+from application.functions.sentiment import sentiment_prediction, count_predictions, clear_data, scrapp_comments
 
 sentiment_service = Blueprint("sentiment_service", __name__)    
 
@@ -18,4 +18,11 @@ def get_data():
         predicitons[i]['sentence'] = data['sentences'][i]
         predicitons[i]['label'] = predicitons[i]['label'][6:]
 
-    return jsonify({"predictions": predicitons, "count_labels": count_labels})
+    return jsonify({"predictions": predicitons, "count_labels": count_labels})
+
+
+@sentiment_service.route("/scrapp_comments", methods=['POST'])
+def scrapp():
+    url = request.get_json()
+    comments = scrapp_comments(url['link'])
+    return jsonify({'comments':comments})