68 lines
2.1 KiB
Python
68 lines
2.1 KiB
Python
import os
|
|
import pandas as pd
|
|
import jsgf
|
|
from unidecode import unidecode
|
|
import string
|
|
from collections import defaultdict
|
|
|
|
|
|
def decode_prompt(prompt):
|
|
prompt_decoded = unidecode(prompt)
|
|
translator = str.maketrans('', '', string.punctuation)
|
|
prompt_decoded = prompt_decoded.translate(translator)
|
|
return prompt_decoded
|
|
|
|
grammar = jsgf.parse_grammar_file('book.jsgf')
|
|
data_files = []
|
|
|
|
for filename in os.listdir("data"):
|
|
f = os.path.join("data", filename)
|
|
if os.path.isfile(f):
|
|
data_files.append(pd.read_csv(f, sep='\t', header=None))
|
|
|
|
true_positives = 0
|
|
false_positives = 0
|
|
false_negatives = 0
|
|
|
|
acts_recognized = defaultdict(int)
|
|
acts_not_recognized = defaultdict(int)
|
|
false_negatives = 0
|
|
false_positives = 0
|
|
|
|
for df in data_files:
|
|
if len(df.columns) == 3:
|
|
df.columns = ["agent", "message", "act"]
|
|
elif len(df.columns) == 2:
|
|
df.columns = ["agent", "message"]
|
|
else:
|
|
continue
|
|
|
|
user_speech_rows = df[df['agent'] == "user"]
|
|
user_speeches = user_speech_rows["message"]
|
|
entries_count = len(user_speeches)
|
|
|
|
found_rules = user_speeches.apply(lambda x: grammar.find_matching_rules(decode_prompt(x)))
|
|
|
|
for line, rules in zip(user_speech_rows.iterrows(), found_rules):
|
|
act = line[1]['act'].split('(')[0]
|
|
if len(rules) > 0:
|
|
recognized_act = rules[0].name
|
|
if recognized_act in act:
|
|
true_positives += 1
|
|
acts_recognized[act] += 1
|
|
else:
|
|
false_positives += 1
|
|
acts_not_recognized[act] += 1
|
|
else:
|
|
false_negatives += 1
|
|
acts_not_recognized[act] += 1
|
|
|
|
accuracy = (true_positives + false_positives) / ((true_positives + false_positives) + sum([x for x in acts_not_recognized.values()]))
|
|
precision = true_positives / (true_positives + false_positives) if (true_positives + false_positives) != 0 else 0
|
|
recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) != 0 else 0
|
|
|
|
print(f"Accuracy: {accuracy}")
|
|
print(f"Precision: {precision}")
|
|
print(f"Recall: {recall}")
|
|
|