New edition of testing script

musique/interpreter/builtin_functions.cc

@@ -188,8 +188,8 @@ static Result<Value> builtin_range(Interpreter&, std::vector<Value> args)
 			array.elements.push_back(start);
 		}
 	} else {
-		for (; start < stop; start += step) {
+		for (; stop > start; stop -= step) {
-			array.elements.push_back(stop - start - Number(1));
+			array.elements.push_back(stop - Number(1));
 		}
 	}
 	return array;

regression-tests/boolean/logical_and.mq

@@ -0,0 +1,14 @@
+say (false and false);
+say (false and true);
+say (true  and false);
+say (true  and true);
+
+-- Test value preservation
+say (0 and 5);
+say (1 and 5);
+say (false and 4);
+say (true  and 4);
+
+-- Test lazy evaluation
+(say 32; false) and (say 42);
+(say 32; true) and (say 42);

regression-tests/boolean/logical_or.mq

@@ -0,0 +1,13 @@
+say (false or false);
+say (false or true);
+say (true or false);
+say (true or true);
+
+-- Test value preservation
+say (0 or 1);
+say (0 or 0);
+say (4 or 3);
+
+-- Test lazy evaluation
+(say 42; false) or say 10;
+(say 42; true) or say 10;

regression-tests/builtin/permute.mq

@@ -0,0 +1,33 @@
+-- 4! = 24 permutations
+say (permute 0 1 2 3);
+say (permute 0 1 3 2);
+say (permute 0 2 1 3);
+say (permute 0 2 3 1);
+say (permute 0 3 1 2);
+say (permute 0 3 2 1);
+say (permute 1 0 2 3);
+say (permute 1 0 3 2);
+say (permute 1 2 0 3);
+say (permute 1 2 3 0);
+say (permute 1 3 0 2);
+say (permute 1 3 2 0);
+say (permute 2 0 1 3);
+say (permute 2 0 3 1);
+say (permute 2 1 0 3);
+say (permute 2 1 3 0);
+say (permute 2 3 0 1);
+say (permute 2 3 1 0);
+say (permute 3 0 1 2);
+say (permute 3 0 2 1);
+say (permute 3 1 0 2);
+say (permute 3 1 2 0);
+say (permute 3 2 0 1);
+say (permute 3 2 1 0);
+
+
+-- This array should be flattened to support case (permute array)
+say (permute 3 2 [1; 0]);
+
+-- Test if nested arrays are preserved
+say (permute [[3; 2]; 4] [1; 0]);
+say (permute [0; 1; 4; [3; 2]]);

regression-tests/builtin/range.mq

@@ -0,0 +1,17 @@
+say (range 0);
+say (range (0 - 1));
+say (range 10);
+say (range 1 10);
+say (range 1 10 2);
+
+say (up 0);
+say (up (0 - 1));
+say (up 10);
+say (up 1 10);
+say (up 1 10 2);
+
+say (down 0);
+say (down (0 - 1));
+say (down 10);
+say (down 1 10);
+say (down 1 10 2);

regression-tests/test_db.json

@@ -0,0 +1,107 @@
+[
+  {
+    "name": "boolean",
+    "cases": [
+      {
+        "name": "logical_or.mq",
+        "exit_code": 0,
+        "stdin_lines": [],
+        "stdout_lines": [
+          "false",
+          "true",
+          "true",
+          "true",
+          "1",
+          "0",
+          "4",
+          "42",
+          "10",
+          "42"
+        ],
+        "stderr_lines": []
+      },
+      {
+        "name": "logical_and.mq",
+        "exit_code": 0,
+        "stdin_lines": [],
+        "stdout_lines": [
+          "false",
+          "false",
+          "false",
+          "true",
+          "0",
+          "5",
+          "false",
+          "4",
+          "32",
+          "32",
+          "42"
+        ],
+        "stderr_lines": []
+      }
+    ]
+  },
+  {
+    "name": "builtin",
+    "cases": [
+      {
+        "name": "permute.mq",
+        "exit_code": 0,
+        "stdin_lines": [],
+        "stdout_lines": [
+          "[0; 1; 3; 2]",
+          "[0; 2; 1; 3]",
+          "[0; 2; 3; 1]",
+          "[0; 3; 1; 2]",
+          "[0; 3; 2; 1]",
+          "[1; 0; 2; 3]",
+          "[1; 0; 3; 2]",
+          "[1; 2; 0; 3]",
+          "[1; 2; 3; 0]",
+          "[1; 3; 0; 2]",
+          "[1; 3; 2; 0]",
+          "[2; 0; 1; 3]",
+          "[2; 0; 3; 1]",
+          "[2; 1; 0; 3]",
+          "[2; 1; 3; 0]",
+          "[2; 3; 0; 1]",
+          "[2; 3; 1; 0]",
+          "[3; 0; 1; 2]",
+          "[3; 0; 2; 1]",
+          "[3; 1; 0; 2]",
+          "[3; 1; 2; 0]",
+          "[3; 2; 0; 1]",
+          "[3; 2; 1; 0]",
+          "[0; 1; 2; 3]",
+          "[0; 1; 2; 3]",
+          "[0; 1; 4; [3; 2]]",
+          "[0; 4; [3; 2]; 1]"
+        ],
+        "stderr_lines": []
+      },
+      {
+        "name": "range.mq",
+        "exit_code": 0,
+        "stdin_lines": [],
+        "stdout_lines": [
+          "[]",
+          "[]",
+          "[0; 1; 2; 3; 4; 5; 6; 7; 8; 9]",
+          "[1; 2; 3; 4; 5; 6; 7; 8; 9]",
+          "[1; 3; 5; 7; 9]",
+          "[]",
+          "[]",
+          "[0; 1; 2; 3; 4; 5; 6; 7; 8; 9]",
+          "[1; 2; 3; 4; 5; 6; 7; 8; 9]",
+          "[1; 3; 5; 7; 9]",
+          "[]",
+          "[]",
+          "[9; 8; 7; 6; 5; 4; 3; 2; 1; 0]",
+          "[9; 8; 7; 6; 5; 4; 3; 2; 1]",
+          "[9; 7; 5; 3; 1]"
+        ],
+        "stderr_lines": []
+      }
+    ]
+  }
+]

scripts/test.mk

@@ -1,3 +1,3 @@
 test: bin/debug/musique
-	scripts/test.py test examples
+	python3 scripts/test.py
 

scripts/test.py

@@ -1,170 +1,160 @@
-#!/usr/bin/env python3
+import argparse
-from dataclasses import dataclass
+import dataclasses
-from glob import glob
-from sys import argv
-from sys import exit
-import os.path
-import shlex
-import subprocess
 import json
-from unittest import case
+import os
+import subprocess
 
-Interpreter = "bin/musique"
+TEST_DB = "test_db.json"
+INTERPRETER = "bin/debug/musique"
 
-def directories_in_path(path: str):
+@dataclasses.dataclass
-    dirs = []
+class Result:
-    while True:
+    exit_code:    int       = 0
-        dirname, _ = os.path.split(path)
+    stdin_lines:  list[str] = dataclasses.field(default_factory=list)
-        if not dirname:
+    stdout_lines: list[str] = dataclasses.field(default_factory=list)
-            break
+    stderr_lines: list[str] = dataclasses.field(default_factory=list)
-        dirs.append(dirname)
-        path = dirname
-        if path == "/":
-            break
-    dirs.reverse()
-    return dirs
 
-def mkdir_recursive(path: str):
+@dataclasses.dataclass
-    for directory in directories_in_path(path):
+class TestCase:
-        try:
+    name:         str
-            os.mkdir(directory)
+    exit_code:    int       = 0
-        except FileExistsError:
+    stdin_lines:  list[str] = dataclasses.field(default_factory=list)
+    stdout_lines: list[str] = dataclasses.field(default_factory=list)
+    stderr_lines: list[str] = dataclasses.field(default_factory=list)
+
+    def run(self, interpreter: str, source: str, cwd: str):
+        result = subprocess.run(
+            args=[interpreter, source],
+            capture_output=True,
+            cwd=cwd,
+            text=True
+        )
+        return Result(
+            exit_code=result.returncode,
+            stdout_lines=result.stdout.splitlines(keepends=False),
+            stderr_lines=result.stderr.splitlines(keepends=False)
+        )
+
+    def record(self, interpreter: str, source: str, cwd: str):
+        print(f"Recording case {self.name}")
+        result = self.run(interpreter, source, cwd)
+
+        changes = []
+        if self.exit_code    != result.exit_code:    changes.append("exit code")
+        if self.stderr_lines != result.stderr_lines: changes.append("stderr")
+        if self.stdout_lines != result.stdout_lines: changes.append("stdout")
+        if changes:
+            print(f"  changed: {', '.join(changes)}")
+
+        self.exit_code, self.stderr_lines, self.stdout_lines = result.exit_code, result.stderr_lines, result.stdout_lines
+
+    def test(self, interpreter: str, source: str, cwd: str):
+        print(f"  Testing case {self.name}  ", end="")
+        result = self.run(interpreter, source, cwd)
+        if self.exit_code == result.exit_code and self.stdout_lines == result.stdout_lines and self.stderr_lines == result.stderr_lines:
+            print("ok")
+            return True
+
+        print(f"FAILED")
+        print(f"File: {source}")
+
+        if self.exit_code != result.exit_code:
+            print(f"Different exit code - expected {self.exit_code}, got {result.exit_code}")
+
+        for name, expected, actual in [
+            ("standard output", self.stdout_lines, result.stdout_lines),
+            ("standard error",  self.stderr_lines, result.stderr_lines)
+        ]:
+            if expected == actual:
                 continue
 
-@dataclass
+            diff_line = None
-class Test_Case:
+            for i, (exp_line, got_line) in enumerate(zip(expected, actual)):
-    returncode = 0
+                if exp_line != got_line:
+                    diff_line = i
+                    break
 
-    @staticmethod
+            if diff_line is not None:
-    def from_file(fname : str):
+                print(f"First difference at line {diff_line+1} in {name}:")
-        with open(fname) as f:
+                print(f"  Expected: {expected[diff_line]}")
-            content = json.load(f)
+                print(f"       Got: {actual[diff_line]}")
-            tc = Test_Case()
+            elif len(expected) > len(actual):
-            for name in content:
+                print(f"Expected {name} is {len(expected) - len(actual)} lines longer then actual")
-                # assert hasattr(tc, name), "Test_Case does not have attribute %s" % (name,)
-                setattr(tc, name, content[name])
-            return tc
-
-    @staticmethod
-    def from_run(run, flags=[]):
-        tc = Test_Case()
-        for attr in ["returncode", "stdout", "stderr"]: ### TODO FLAGS
-            try:
-                run_attr = getattr(run, attr).decode()
-            except (UnicodeDecodeError, AttributeError):
-                run_attr = getattr(run, attr)
-
-            setattr(tc, attr, run_attr)
-        setattr(tc, "flags", flags)
-        return tc
-
-    def save(self, fname : str):
-        j = {}
-        for attr in ["returncode", "stdout", "stderr", "flags"]:
-            j[attr] = getattr(self, attr)
-
-        mkdir_recursive(fname)
-        with open(fname, 'w') as f:
-            json.dump(j, f, indent=4)
-
-def cmd_run_echoed(cmd, **kwargs):
-    print("[CMD] %s" % " ".join(map(shlex.quote, cmd)))
-    return subprocess.run(cmd, **kwargs)
-
-def find_path_for_test_case(path: str) -> str:
-    directory, filename = os.path.split(path)
-    return (directory if directory else ".") + "/.tests_cache/" + filename + ".json"
-
-def run_tests(file_paths: list):
-    return_code = 0
-    for program_file in file_paths:
-        test_case_file = find_path_for_test_case(program_file)
-        if os.path.exists(test_case_file):
-            tc = Test_Case.from_file(test_case_file)
             else:
-            continue
+                print(f"Actual {name} is {len(actual) - len(expected)} lines longer then expected")
 
-        flags_list = [Interpreter]
+        return False
-        if hasattr(tc, "flags"):
-            flags_list.extend(tc.flags)
-        flags_list.append(program_file)
-
-        res = cmd_run_echoed(flags_list, capture_output=True)
-
-        for attr in [a for a in dir(tc) if a in ["returncode", "stdout", "stderr"]]:
-            tc_attr = getattr(tc, attr)
-            res_attr = getattr(res, attr)
-            try:
-                res_attr = res_attr.decode()
-            except (UnicodeDecodeError, AttributeError):
-                pass
-
-            if tc_attr != res_attr:
-                print(f"[ERROR] Failed test {program_file}")
-                print(f"Expected {attr} = ")
-                print(tc_attr)
-                print(f"Received {attr} = ")
-                print(res_attr)
-                return_code = 1
-    exit(return_code)
-
-def record_tests(file_paths: list):
-    for program_file in file_paths:
-        test_case_file = find_path_for_test_case(program_file)
-
-        res = cmd_run_echoed([Interpreter, program_file], capture_output=True)
-        tc = Test_Case.from_run(res, [])
-        tc.save(test_case_file)
-
-def add_tests(file_paths: list):
-    to_be_added = []
-
-    for program_file in file_paths:
-        test_case_file = find_path_for_test_case(program_file)
-        if not os.path.exists(test_case_file):
-            print(f"Add test {program_file}? (yes/no)")
-            if "yes".startswith(input().strip().lower()):
-                to_be_added.append(program_file)
-
-    record_tests(to_be_added)
 
 
-# list of files to test
+@dataclasses.dataclass
-def main():
+class TestSuite:
-    file_paths, mode = [], run_tests
+    name:  str
+    cases: list[TestCase] = dataclasses.field(default_factory=list)
 
-    if len(argv) < 2:
+suites = list[TestSuite]()
-        print("[ERROR] Expected mode argument (either 'record', 'add' or 'test')")
-        exit(1)
 
-    if argv[1] == "test":
+def traverse(discover: bool, update: bool):
-        mode = run_tests
+    to_record = list[tuple[TestSuite, TestCase]]()
-    elif argv[1] == "record":
+    if discover:
-        mode = record_tests
+        for suite_name in os.listdir(testing_dir):
-    elif argv[1] == "add":
+            if os.path.isdir(os.path.join(testing_dir, suite_name)) and suite_name not in (suite.name for suite in suites):
-        mode = add_tests
+                print(f"Discovered new test suite: {suite_name}")
-    else:
+                suites.append(TestSuite(name=suite_name))
-        print(f"[ERROR] Unrecognized mode '{argv[1]}'")
-        exit(1)
 
-    if len(argv) < 3:
+        for suite in suites:
-        print("[ERROR] Expected test case")
+            suite_path = os.path.join(testing_dir, suite.name)
-        exit(1)
+            for case_name in os.listdir(suite_path):
+                if os.path.isfile(os.path.join(suite_path, case_name)) and case_name not in (case.name for case in suite.cases):
+                    print(f"In suite '{suite.name}' discovered new test case: {case_name}")
+                    case = TestCase(name=case_name)
+                    suite.cases.append(case)
+                    to_record.append((suite, case))
 
-    interpreter = os.getenv("INTERPRETER")
+    if update:
-    if interpreter:
+        to_record.extend(((suite, case) for suite in suites for case in suite.cases))
-        Interpreter = interpreter
 
-    for path in argv[2:]:
+    for (suite, case) in to_record:
-        if os.path.exists(path):
+        case.record(
-            if os.path.isdir(path):
+            interpreter=os.path.join(root, INTERPRETER),
-                file_paths.extend(glob(f"{path}/*.mq"))
+            source=os.path.join(testing_dir, suite.name, case.name),
-            else:
+            cwd=root
-                file_paths.append(path)
+        )
-        elif mode == add_tests:
-            print("Adding: " + path)
 
-    mode(file_paths)
+    with open(test_db_path, "w") as f:
+        json_suites = [dataclasses.asdict(suite) for suite in suites]
+        json.dump(json_suites, f, indent=2)
+
+def test():
+    successful, total = 0, 0
+    for suite in suites:
+        print(f"Testing suite {suite.name}")
+        for case in suite.cases:
+            successful += int(case.test(
+                interpreter=os.path.join(root, INTERPRETER),
+                source=os.path.join(testing_dir, suite.name, case.name),
+                cwd=root
+            ))
+            total += 1
+
+    print(f"Passed {successful} out of {total} ({100 * successful // total}%)")
 
 if __name__ == "__main__":
-    main()
+    parser = argparse.ArgumentParser(description="Regression test runner for Musique programming language")
+    parser.add_argument("-d", "--discover", action="store_true", help="Discover all tests that are not in testing database")
+    parser.add_argument("-u", "--update", action="store_true", help="Update all tests")
+
+    args = parser.parse_args()
+
+    root = os.path.dirname(os.path.dirname(__file__))
+    testing_dir = os.path.join(root, "regression-tests")
+    test_db_path = os.path.join(testing_dir, TEST_DB)
+
+    with open(test_db_path, "r") as f:
+        for src in json.load(f):
+            src["cases"] = [TestCase(**case) for case in src["cases"]]
+            suites.append(TestSuite(**src))
+
+    if args.discover or args.update:
+        traverse(discover=args.discover, update=args.update)
+    else:
+        test()