diff --git a/UserActType.py b/UserActType.py index 599f59a..59db987 100644 --- a/UserActType.py +++ b/UserActType.py @@ -10,5 +10,6 @@ class UserActType(Enum): CHANGE_MEETING = 4 MEETING_LIST = 5 CONFIRM = 6 - THANKYOU = 7 + NEGATE = 7 + THANKYOU = 8 INVALID = -1 diff --git a/book.jsgf b/book.jsgf index 2beafdf..1ed1a52 100644 --- a/book.jsgf +++ b/book.jsgf @@ -22,6 +22,8 @@ public = potwierdzam | tak | zgadza się; public = witam | dzień dobry | cześć | siemka | dobry wieczór | hej | hejka; -public = dziękuję (bardzo)*; +public = (bardzo)* dziękuję (bardzo)*; -public = do widzenia | miłego dnia | to by było wszystko na teraz | to wszystko; \ No newline at end of file +public = do widzenia | miłego dnia | to by było wszystko na teraz | to wszystko | to wszystko na tę chwilę; + +public = nie; \ No newline at end of file diff --git a/evaluate.py b/evaluate.py new file mode 100644 index 0000000..b3b87e1 --- /dev/null +++ b/evaluate.py @@ -0,0 +1,30 @@ +import os +import re +import jsgf +import pandas as pd +import numpy as np +from NaturalLanguageUnderstanding import NLU + +nlu = NLU() + +rows = 0 +hits = 0 + +for file_name in os.listdir('data'): + df = pd.read_csv(f'data/{file_name}', sep='\t', names=['interlocutor', 'sentence', 'acts']) + df = df[df.interlocutor == 'user'] + data = np.array(df) + + for row in data: + sentence = row[1] + sentences = sentence.split(',') + for sentence in sentences: + acts = row[2].split('&') + for act in acts: + rows += 1 + frame = nlu.parse_user_input(sentence) + frame_act_name = frame.getActType().name.lower() + if frame_act_name in act.strip(): + hits += 1 + +print(f"Accuracy: {(hits / rows)*100}")