From 469e437f3d8d6b36ea50a80bbe4e4c5035c0fd9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20J=C4=99dyk?= <lukasz.jedyk@gmail.com>
Date: Mon, 17 May 2021 15:06:42 +0200
Subject: [PATCH] grammar changes + evaluate

---
 UserActType.py |  3 ++-
 book.jsgf      |  6 ++++--
 evaluate.py    | 30 ++++++++++++++++++++++++++++++
 3 files changed, 36 insertions(+), 3 deletions(-)
 create mode 100644 evaluate.py

diff --git a/UserActType.py b/UserActType.py
index 599f59a..59db987 100644
--- a/UserActType.py
+++ b/UserActType.py
@@ -10,5 +10,6 @@ class UserActType(Enum):
     CHANGE_MEETING = 4
     MEETING_LIST = 5
     CONFIRM = 6
-    THANKYOU = 7
+    NEGATE = 7
+    THANKYOU = 8
     INVALID = -1
diff --git a/book.jsgf b/book.jsgf
index 2beafdf..1ed1a52 100644
--- a/book.jsgf
+++ b/book.jsgf
@@ -22,6 +22,8 @@ public <confirm> = potwierdzam | tak | zgadza się;
 
 public <hello> = witam | dzień dobry | cześć | siemka | dobry wieczór | hej | hejka;
 
-public <thankyou> = dziękuję (bardzo)*;
+public <thankyou> = (bardzo)* dziękuję (bardzo)*;
 
-public <bye> = do widzenia | miłego dnia | to by było wszystko na teraz | <thankyou> to wszystko;
\ No newline at end of file
+public <bye> = do widzenia | miłego dnia | to by było wszystko na teraz | <thankyou> to wszystko | to wszystko na tę chwilę;
+
+public <negate> = nie; 
\ No newline at end of file
diff --git a/evaluate.py b/evaluate.py
new file mode 100644
index 0000000..b3b87e1
--- /dev/null
+++ b/evaluate.py
@@ -0,0 +1,30 @@
+import os
+import re
+import jsgf
+import pandas as pd
+import numpy as np
+from NaturalLanguageUnderstanding import NLU
+
+nlu = NLU()
+
+rows = 0
+hits = 0
+
+for file_name in os.listdir('data'):
+    df = pd.read_csv(f'data/{file_name}', sep='\t', names=['interlocutor', 'sentence', 'acts'])
+    df = df[df.interlocutor == 'user']
+    data = np.array(df)
+
+    for row in data:
+        sentence = row[1]
+        sentences = sentence.split(',')
+        for sentence in sentences:
+            acts = row[2].split('&')
+            for act in acts:
+                rows += 1
+                frame = nlu.parse_user_input(sentence)
+                frame_act_name = frame.getActType().name.lower()
+                if frame_act_name in act.strip():
+                    hits += 1
+
+print(f"Accuracy: {(hits / rows)*100}")