From 548b29d897bb1d25ac4b10d1a4a8b33a69802a90 Mon Sep 17 00:00:00 2001 From: Robert Bendun Date: Sun, 25 Sep 2022 03:19:00 +0200 Subject: [PATCH] New edition of testing script --- musique/interpreter/builtin_functions.cc | 4 +- regression-tests/boolean/logical_and.mq | 14 ++ regression-tests/boolean/logical_or.mq | 13 + regression-tests/builtin/permute.mq | 33 +++ regression-tests/builtin/range.mq | 17 ++ regression-tests/test_db.json | 107 +++++++++ scripts/test.mk | 2 +- scripts/test.py | 294 +++++++++++------------ 8 files changed, 329 insertions(+), 155 deletions(-) create mode 100644 regression-tests/boolean/logical_and.mq create mode 100644 regression-tests/boolean/logical_or.mq create mode 100644 regression-tests/builtin/permute.mq create mode 100644 regression-tests/builtin/range.mq create mode 100644 regression-tests/test_db.json mode change 100755 => 100644 scripts/test.py diff --git a/musique/interpreter/builtin_functions.cc b/musique/interpreter/builtin_functions.cc index 62b07c3..3fd54e0 100644 --- a/musique/interpreter/builtin_functions.cc +++ b/musique/interpreter/builtin_functions.cc @@ -188,8 +188,8 @@ static Result builtin_range(Interpreter&, std::vector args) array.elements.push_back(start); } } else { - for (; start < stop; start += step) { - array.elements.push_back(stop - start - Number(1)); + for (; stop > start; stop -= step) { + array.elements.push_back(stop - Number(1)); } } return array; diff --git a/regression-tests/boolean/logical_and.mq b/regression-tests/boolean/logical_and.mq new file mode 100644 index 0000000..39ae3f7 --- /dev/null +++ b/regression-tests/boolean/logical_and.mq @@ -0,0 +1,14 @@ +say (false and false); +say (false and true); +say (true and false); +say (true and true); + +-- Test value preservation +say (0 and 5); +say (1 and 5); +say (false and 4); +say (true and 4); + +-- Test lazy evaluation +(say 32; false) and (say 42); +(say 32; true) and (say 42); diff --git a/regression-tests/boolean/logical_or.mq b/regression-tests/boolean/logical_or.mq new file mode 100644 index 0000000..e7a15ba --- /dev/null +++ b/regression-tests/boolean/logical_or.mq @@ -0,0 +1,13 @@ +say (false or false); +say (false or true); +say (true or false); +say (true or true); + +-- Test value preservation +say (0 or 1); +say (0 or 0); +say (4 or 3); + +-- Test lazy evaluation +(say 42; false) or say 10; +(say 42; true) or say 10; diff --git a/regression-tests/builtin/permute.mq b/regression-tests/builtin/permute.mq new file mode 100644 index 0000000..16ac955 --- /dev/null +++ b/regression-tests/builtin/permute.mq @@ -0,0 +1,33 @@ +-- 4! = 24 permutations +say (permute 0 1 2 3); +say (permute 0 1 3 2); +say (permute 0 2 1 3); +say (permute 0 2 3 1); +say (permute 0 3 1 2); +say (permute 0 3 2 1); +say (permute 1 0 2 3); +say (permute 1 0 3 2); +say (permute 1 2 0 3); +say (permute 1 2 3 0); +say (permute 1 3 0 2); +say (permute 1 3 2 0); +say (permute 2 0 1 3); +say (permute 2 0 3 1); +say (permute 2 1 0 3); +say (permute 2 1 3 0); +say (permute 2 3 0 1); +say (permute 2 3 1 0); +say (permute 3 0 1 2); +say (permute 3 0 2 1); +say (permute 3 1 0 2); +say (permute 3 1 2 0); +say (permute 3 2 0 1); +say (permute 3 2 1 0); + + +-- This array should be flattened to support case (permute array) +say (permute 3 2 [1; 0]); + +-- Test if nested arrays are preserved +say (permute [[3; 2]; 4] [1; 0]); +say (permute [0; 1; 4; [3; 2]]); diff --git a/regression-tests/builtin/range.mq b/regression-tests/builtin/range.mq new file mode 100644 index 0000000..959cf2e --- /dev/null +++ b/regression-tests/builtin/range.mq @@ -0,0 +1,17 @@ +say (range 0); +say (range (0 - 1)); +say (range 10); +say (range 1 10); +say (range 1 10 2); + +say (up 0); +say (up (0 - 1)); +say (up 10); +say (up 1 10); +say (up 1 10 2); + +say (down 0); +say (down (0 - 1)); +say (down 10); +say (down 1 10); +say (down 1 10 2); diff --git a/regression-tests/test_db.json b/regression-tests/test_db.json new file mode 100644 index 0000000..43e6bb6 --- /dev/null +++ b/regression-tests/test_db.json @@ -0,0 +1,107 @@ +[ + { + "name": "boolean", + "cases": [ + { + "name": "logical_or.mq", + "exit_code": 0, + "stdin_lines": [], + "stdout_lines": [ + "false", + "true", + "true", + "true", + "1", + "0", + "4", + "42", + "10", + "42" + ], + "stderr_lines": [] + }, + { + "name": "logical_and.mq", + "exit_code": 0, + "stdin_lines": [], + "stdout_lines": [ + "false", + "false", + "false", + "true", + "0", + "5", + "false", + "4", + "32", + "32", + "42" + ], + "stderr_lines": [] + } + ] + }, + { + "name": "builtin", + "cases": [ + { + "name": "permute.mq", + "exit_code": 0, + "stdin_lines": [], + "stdout_lines": [ + "[0; 1; 3; 2]", + "[0; 2; 1; 3]", + "[0; 2; 3; 1]", + "[0; 3; 1; 2]", + "[0; 3; 2; 1]", + "[1; 0; 2; 3]", + "[1; 0; 3; 2]", + "[1; 2; 0; 3]", + "[1; 2; 3; 0]", + "[1; 3; 0; 2]", + "[1; 3; 2; 0]", + "[2; 0; 1; 3]", + "[2; 0; 3; 1]", + "[2; 1; 0; 3]", + "[2; 1; 3; 0]", + "[2; 3; 0; 1]", + "[2; 3; 1; 0]", + "[3; 0; 1; 2]", + "[3; 0; 2; 1]", + "[3; 1; 0; 2]", + "[3; 1; 2; 0]", + "[3; 2; 0; 1]", + "[3; 2; 1; 0]", + "[0; 1; 2; 3]", + "[0; 1; 2; 3]", + "[0; 1; 4; [3; 2]]", + "[0; 4; [3; 2]; 1]" + ], + "stderr_lines": [] + }, + { + "name": "range.mq", + "exit_code": 0, + "stdin_lines": [], + "stdout_lines": [ + "[]", + "[]", + "[0; 1; 2; 3; 4; 5; 6; 7; 8; 9]", + "[1; 2; 3; 4; 5; 6; 7; 8; 9]", + "[1; 3; 5; 7; 9]", + "[]", + "[]", + "[0; 1; 2; 3; 4; 5; 6; 7; 8; 9]", + "[1; 2; 3; 4; 5; 6; 7; 8; 9]", + "[1; 3; 5; 7; 9]", + "[]", + "[]", + "[9; 8; 7; 6; 5; 4; 3; 2; 1; 0]", + "[9; 8; 7; 6; 5; 4; 3; 2; 1]", + "[9; 7; 5; 3; 1]" + ], + "stderr_lines": [] + } + ] + } +] \ No newline at end of file diff --git a/scripts/test.mk b/scripts/test.mk index b8e2c33..23df39a 100644 --- a/scripts/test.mk +++ b/scripts/test.mk @@ -1,3 +1,3 @@ test: bin/debug/musique - scripts/test.py test examples + python3 scripts/test.py diff --git a/scripts/test.py b/scripts/test.py old mode 100755 new mode 100644 index 82934f2..fdd4560 --- a/scripts/test.py +++ b/scripts/test.py @@ -1,170 +1,160 @@ -#!/usr/bin/env python3 -from dataclasses import dataclass -from glob import glob -from sys import argv -from sys import exit -import os.path -import shlex -import subprocess +import argparse +import dataclasses import json -from unittest import case +import os +import subprocess -Interpreter = "bin/musique" +TEST_DB = "test_db.json" +INTERPRETER = "bin/debug/musique" -def directories_in_path(path: str): - dirs = [] - while True: - dirname, _ = os.path.split(path) - if not dirname: - break - dirs.append(dirname) - path = dirname - if path == "/": - break - dirs.reverse() - return dirs +@dataclasses.dataclass +class Result: + exit_code: int = 0 + stdin_lines: list[str] = dataclasses.field(default_factory=list) + stdout_lines: list[str] = dataclasses.field(default_factory=list) + stderr_lines: list[str] = dataclasses.field(default_factory=list) -def mkdir_recursive(path: str): - for directory in directories_in_path(path): - try: - os.mkdir(directory) - except FileExistsError: - continue +@dataclasses.dataclass +class TestCase: + name: str + exit_code: int = 0 + stdin_lines: list[str] = dataclasses.field(default_factory=list) + stdout_lines: list[str] = dataclasses.field(default_factory=list) + stderr_lines: list[str] = dataclasses.field(default_factory=list) -@dataclass -class Test_Case: - returncode = 0 + def run(self, interpreter: str, source: str, cwd: str): + result = subprocess.run( + args=[interpreter, source], + capture_output=True, + cwd=cwd, + text=True + ) + return Result( + exit_code=result.returncode, + stdout_lines=result.stdout.splitlines(keepends=False), + stderr_lines=result.stderr.splitlines(keepends=False) + ) - @staticmethod - def from_file(fname : str): - with open(fname) as f: - content = json.load(f) - tc = Test_Case() - for name in content: - # assert hasattr(tc, name), "Test_Case does not have attribute %s" % (name,) - setattr(tc, name, content[name]) - return tc + def record(self, interpreter: str, source: str, cwd: str): + print(f"Recording case {self.name}") + result = self.run(interpreter, source, cwd) - @staticmethod - def from_run(run, flags=[]): - tc = Test_Case() - for attr in ["returncode", "stdout", "stderr"]: ### TODO FLAGS - try: - run_attr = getattr(run, attr).decode() - except (UnicodeDecodeError, AttributeError): - run_attr = getattr(run, attr) + changes = [] + if self.exit_code != result.exit_code: changes.append("exit code") + if self.stderr_lines != result.stderr_lines: changes.append("stderr") + if self.stdout_lines != result.stdout_lines: changes.append("stdout") + if changes: + print(f" changed: {', '.join(changes)}") - setattr(tc, attr, run_attr) - setattr(tc, "flags", flags) - return tc + self.exit_code, self.stderr_lines, self.stdout_lines = result.exit_code, result.stderr_lines, result.stdout_lines - def save(self, fname : str): - j = {} - for attr in ["returncode", "stdout", "stderr", "flags"]: - j[attr] = getattr(self, attr) + def test(self, interpreter: str, source: str, cwd: str): + print(f" Testing case {self.name} ", end="") + result = self.run(interpreter, source, cwd) + if self.exit_code == result.exit_code and self.stdout_lines == result.stdout_lines and self.stderr_lines == result.stderr_lines: + print("ok") + return True - mkdir_recursive(fname) - with open(fname, 'w') as f: - json.dump(j, f, indent=4) + print(f"FAILED") + print(f"File: {source}") -def cmd_run_echoed(cmd, **kwargs): - print("[CMD] %s" % " ".join(map(shlex.quote, cmd))) - return subprocess.run(cmd, **kwargs) + if self.exit_code != result.exit_code: + print(f"Different exit code - expected {self.exit_code}, got {result.exit_code}") -def find_path_for_test_case(path: str) -> str: - directory, filename = os.path.split(path) - return (directory if directory else ".") + "/.tests_cache/" + filename + ".json" + for name, expected, actual in [ + ("standard output", self.stdout_lines, result.stdout_lines), + ("standard error", self.stderr_lines, result.stderr_lines) + ]: + if expected == actual: + continue -def run_tests(file_paths: list): - return_code = 0 - for program_file in file_paths: - test_case_file = find_path_for_test_case(program_file) - if os.path.exists(test_case_file): - tc = Test_Case.from_file(test_case_file) - else: - continue + diff_line = None + for i, (exp_line, got_line) in enumerate(zip(expected, actual)): + if exp_line != got_line: + diff_line = i + break - flags_list = [Interpreter] - if hasattr(tc, "flags"): - flags_list.extend(tc.flags) - flags_list.append(program_file) - - res = cmd_run_echoed(flags_list, capture_output=True) - - for attr in [a for a in dir(tc) if a in ["returncode", "stdout", "stderr"]]: - tc_attr = getattr(tc, attr) - res_attr = getattr(res, attr) - try: - res_attr = res_attr.decode() - except (UnicodeDecodeError, AttributeError): - pass - - if tc_attr != res_attr: - print(f"[ERROR] Failed test {program_file}") - print(f"Expected {attr} = ") - print(tc_attr) - print(f"Received {attr} = ") - print(res_attr) - return_code = 1 - exit(return_code) - -def record_tests(file_paths: list): - for program_file in file_paths: - test_case_file = find_path_for_test_case(program_file) - - res = cmd_run_echoed([Interpreter, program_file], capture_output=True) - tc = Test_Case.from_run(res, []) - tc.save(test_case_file) - -def add_tests(file_paths: list): - to_be_added = [] - - for program_file in file_paths: - test_case_file = find_path_for_test_case(program_file) - if not os.path.exists(test_case_file): - print(f"Add test {program_file}? (yes/no)") - if "yes".startswith(input().strip().lower()): - to_be_added.append(program_file) - - record_tests(to_be_added) - - -# list of files to test -def main(): - file_paths, mode = [], run_tests - - if len(argv) < 2: - print("[ERROR] Expected mode argument (either 'record', 'add' or 'test')") - exit(1) - - if argv[1] == "test": - mode = run_tests - elif argv[1] == "record": - mode = record_tests - elif argv[1] == "add": - mode = add_tests - else: - print(f"[ERROR] Unrecognized mode '{argv[1]}'") - exit(1) - - if len(argv) < 3: - print("[ERROR] Expected test case") - exit(1) - - interpreter = os.getenv("INTERPRETER") - if interpreter: - Interpreter = interpreter - - for path in argv[2:]: - if os.path.exists(path): - if os.path.isdir(path): - file_paths.extend(glob(f"{path}/*.mq")) + if diff_line is not None: + print(f"First difference at line {diff_line+1} in {name}:") + print(f" Expected: {expected[diff_line]}") + print(f" Got: {actual[diff_line]}") + elif len(expected) > len(actual): + print(f"Expected {name} is {len(expected) - len(actual)} lines longer then actual") else: - file_paths.append(path) - elif mode == add_tests: - print("Adding: " + path) + print(f"Actual {name} is {len(actual) - len(expected)} lines longer then expected") - mode(file_paths) + return False + + +@dataclasses.dataclass +class TestSuite: + name: str + cases: list[TestCase] = dataclasses.field(default_factory=list) + +suites = list[TestSuite]() + +def traverse(discover: bool, update: bool): + to_record = list[tuple[TestSuite, TestCase]]() + if discover: + for suite_name in os.listdir(testing_dir): + if os.path.isdir(os.path.join(testing_dir, suite_name)) and suite_name not in (suite.name for suite in suites): + print(f"Discovered new test suite: {suite_name}") + suites.append(TestSuite(name=suite_name)) + + for suite in suites: + suite_path = os.path.join(testing_dir, suite.name) + for case_name in os.listdir(suite_path): + if os.path.isfile(os.path.join(suite_path, case_name)) and case_name not in (case.name for case in suite.cases): + print(f"In suite '{suite.name}' discovered new test case: {case_name}") + case = TestCase(name=case_name) + suite.cases.append(case) + to_record.append((suite, case)) + + if update: + to_record.extend(((suite, case) for suite in suites for case in suite.cases)) + + for (suite, case) in to_record: + case.record( + interpreter=os.path.join(root, INTERPRETER), + source=os.path.join(testing_dir, suite.name, case.name), + cwd=root + ) + + with open(test_db_path, "w") as f: + json_suites = [dataclasses.asdict(suite) for suite in suites] + json.dump(json_suites, f, indent=2) + +def test(): + successful, total = 0, 0 + for suite in suites: + print(f"Testing suite {suite.name}") + for case in suite.cases: + successful += int(case.test( + interpreter=os.path.join(root, INTERPRETER), + source=os.path.join(testing_dir, suite.name, case.name), + cwd=root + )) + total += 1 + + print(f"Passed {successful} out of {total} ({100 * successful // total}%)") if __name__ == "__main__": - main() + parser = argparse.ArgumentParser(description="Regression test runner for Musique programming language") + parser.add_argument("-d", "--discover", action="store_true", help="Discover all tests that are not in testing database") + parser.add_argument("-u", "--update", action="store_true", help="Update all tests") + + args = parser.parse_args() + + root = os.path.dirname(os.path.dirname(__file__)) + testing_dir = os.path.join(root, "regression-tests") + test_db_path = os.path.join(testing_dir, TEST_DB) + + with open(test_db_path, "r") as f: + for src in json.load(f): + src["cases"] = [TestCase(**case) for case in src["cases"]] + suites.append(TestSuite(**src)) + + if args.discover or args.update: + traverse(discover=args.discover, update=args.update) + else: + test()