import os import pandas as pd import jsgf from unidecode import unidecode import string from collections import defaultdict def decode_prompt(prompt): prompt_decoded = unidecode(prompt) translator = str.maketrans('', '', string.punctuation) prompt_decoded = prompt_decoded.translate(translator) return prompt_decoded grammar = jsgf.parse_grammar_file('book.jsgf') data_files = [] for filename in os.listdir("data"): f = os.path.join("data", filename) if os.path.isfile(f): data_files.append(pd.read_csv(f, sep='\t', header=None)) recognized = 0 unrecognized = 0 true_positives = 0 false_positives = 0 false_negatives = 0 acts_recognized = defaultdict(int) acts_not_recognized = defaultdict(int) for df in data_files: if len(df.columns) == 3: df.columns = ["agent", "message", "act"] elif len(df.columns) == 2: df.columns = ["agent", "message"] else: continue user_speech_rows = df[df['agent'] == "user"] user_speeches = user_speech_rows["message"] entries_count = len(user_speeches) parsed = user_speeches.apply( lambda x: bool(grammar.find_matching_rules(decode_prompt(x)))) true_count = parsed.sum() false_count = len(parsed) - true_count recognized += true_count unrecognized += false_count for line, correct in zip(df.iterrows(), parsed): acts_recognized[line[1]['act'].split('(')[0]] += int(correct) acts_not_recognized[line[1]['act'].split('(')[0]] += int(not(correct)) print(f"Recognized user utterances: {recognized}") print(f"Unrecognized user utterances: {unrecognized}") print(f"Accuracy: {recognized/(recognized+unrecognized)}") precision_per_class = {} recall_per_class = {} for act in acts_recognized.keys(): true_positives = acts_recognized[act] false_negatives = acts_not_recognized[act] false_positives = recognized - true_positives precision = true_positives / (true_positives + false_positives) if (true_positives + false_positives) != 0 else 0 recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) != 0 else 0 precision_per_class[act] = precision recall_per_class[act] = recall average_precision = sum(precision_per_class.values()) / len(precision_per_class) average_recall = sum(recall_per_class.values()) / len(recall_per_class) print("\nPrecision per class:") for act, precision in precision_per_class.items(): print(f"{act}: {precision}") print("\nRecall per class:") for act, recall in recall_per_class.items(): print(f"{act}: {recall}") print(f"\nAverage Precision: {average_precision}") print(f"Average Recall: {average_recall}")