s444476/exact_data2023
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix pipelines

Browse Source
This commit is contained in:
Maciej Ścigacz 2023-05-31 16:56:13 +02:00
parent 66a0b1c6bc
commit 51d691f08c
2 changed files with 6 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
application/functions/errors.py
View File

@ -1,10 +1,6 @@
from transformers import AutoTokenizer
from transformers import pipeline from transformers import pipeline
#from transformers import BartForConditionalGeneration
tokenizer = AutoTokenizer.from_pretrained("szymonj/polish-simple-error-correction") pipe = pipeline("text2text-generation",model="szymonj/polish-simple-error-correction", tokenizer="szymonj/polish-simple-error-correction", max_length=2000)
#model = BartForConditionalGeneration.from_pretrained("szymonj/polish-simple-error-correction")
pipe = pipeline("text2text-generation",model="szymonj/polish-simple-error-correction",tokenizer=tokenizer,max_length=2000)
def errors_correction(data): def errors_correction(data):
result = pipe(data) result = pipe(data)

13
application/functions/sentiment.py
View File

@ -1,17 +1,12 @@
from transformers import AutoTokenizer from transformers import pipeline
from transformers import pipeline, GPT2ForSequenceClassification
import re import re
from facebook_scraper import get_posts from facebook_scraper import get_posts
# model = 'application/models/sentiment_model' pipe = pipeline('text-classification', model="Scigi/sentiment-analysis-model", tokenizer = "Scigi/sentiment-analysis-model")
# tokenizer = AutoTokenizer.from_pretrained('application/tokenizers/sentiment_tokenizer')
#model = GPT2ForSequenceClassification.from_pretrained("Scigi/sentiment-analysis-model", num_labels=3)
tokenizer = AutoTokenizer.from_pretrained("Scigi/sentiment-analysis-model")
pipe = pipeline('text-classification', model="Scigi/sentiment-analysis-model", tokenizer = tokenizer)
def sentiment_prediction(data): def sentiment_prediction(data):
result = pipe(data) result = pipe(data)
return result return result
def clear_data(data): def clear_data(data):
@ -19,6 +14,7 @@ def clear_data(data):
data = [x for x in data if x != ''] data = [x for x in data if x != '']
data = [i.strip() for i in data] data = [i.strip() for i in data]
data = [i.lower() for i in data] data = [i.lower() for i in data]
return data return data
def count_predictions(predictions): def count_predictions(predictions):
@ -50,4 +46,5 @@ def scrapp_comments(url):
comments.append(comment['comment_text']) comments.append(comment['comment_text'])
all['post'] = text_post all['post'] = text_post
all['sentences'] = comments all['sentences'] = comments
return all return all