s416496/musique
1
0
Fork 0
Code Issues 3 Pull Requests 1 Projects Releases Wiki Activity

Record and verify musical test cases

Browse Source
This commit is contained in:
Robert Bendun 2023-02-15 01:18:35 +01:00
parent 08cc14e50b
commit cf83432b15
4 changed files with 363 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
musique/main.cc
View File

@ -169,7 +169,7 @@ struct Runner
ensure(the == nullptr, "Only one instance of runner is supported"); ensure(the == nullptr, "Only one instance of runner is supported");
the = this; the = this;
// interpreter.current_context->connect(std::nullopt); interpreter.current_context->connect(std::nullopt);
Env::global->force_define("say", +[](Interpreter &interpreter, std::vector<Value> args) -> Result<Value> { Env::global->force_define("say", +[](Interpreter &interpreter, std::vector<Value> args) -> Result<Value> {
for (auto it = args.begin(); it != args.end(); ++it) { for (auto it = args.begin(); it != args.end(); ++it) {

2
regression-tests/music/all_midi_notes.mq Normal file
View File

@ -0,0 +1,2 @@
len tn,
play (c4 + up 12)

385
regression-tests/test_db.json
View File

@ -3,9 +3,7 @@
"name": "boolean", "name": "boolean",
"cases": [ "cases": [
{ {
"name": "logical_or.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"false", "false",
"true", "true",
@ -18,12 +16,13 @@
"10", "10",
"42" "42"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "logical_or.mq",
"stdin_lines": []
}, },
{ {
"name": "logical_and.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"false", "false",
"false", "false",
@ -37,7 +36,10 @@
"32", "32",
"42" "42"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "logical_and.mq",
"stdin_lines": []
} }
] ]
}, },
@ -45,9 +47,7 @@
"name": "builtin", "name": "builtin",
"cases": [ "cases": [
{ {
"name": "permute.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"(0, 1, 3, 2)", "(0, 1, 3, 2)",
"(0, 2, 1, 3)", "(0, 2, 1, 3)",
@ -77,12 +77,13 @@
"(0, 1, 4, (3, 2))", "(0, 1, 4, (3, 2))",
"(0, 4, (3, 2), 1)" "(0, 4, (3, 2), 1)"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "permute.mq",
"stdin_lines": []
}, },
{ {
"name": "range.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"()", "()",
"()", "()",
@ -100,35 +101,38 @@
"(9, 8, 7, 6, 5, 4, 3, 2, 1)", "(9, 8, 7, 6, 5, 4, 3, 2, 1)",
"(9, 7, 5, 3, 1)" "(9, 7, 5, 3, 1)"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "range.mq",
"stdin_lines": []
}, },
{ {
"name": "min.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"1", "1",
"200", "200",
"100", "100",
"0" "0"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "min.mq",
"stdin_lines": []
}, },
{ {
"name": "call.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"42", "42",
"11", "11",
"43" "43"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "call.mq",
"stdin_lines": []
}, },
{ {
"name": "if.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"1", "1",
"2", "2",
@ -138,35 +142,38 @@
"200", "200",
"9" "9"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "if.mq",
"stdin_lines": []
}, },
{ {
"name": "uniq.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)", "(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)",
"(1, 3, 5, 3, 4, 1)", "(1, 3, 5, 3, 4, 1)",
"(1, 3, 5, 3, 4, 1)" "(1, 3, 5, 3, 4, 1)"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "uniq.mq",
"stdin_lines": []
}, },
{ {
"name": "reverse.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"()", "()",
"(9, 8, 7, 6, 5, 4, 3, 2, 1, 0)", "(9, 8, 7, 6, 5, 4, 3, 2, 1, 0)",
"(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)", "(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)",
"(9, 8, 7, 6, 5, 4, (1, 2, 3))" "(9, 8, 7, 6, 5, 4, (1, 2, 3))"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "reverse.mq",
"stdin_lines": []
}, },
{ {
"name": "typeof.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"array", "array",
"number", "number",
@ -176,35 +183,38 @@
"nil", "nil",
"intrinsic" "intrinsic"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "typeof.mq",
"stdin_lines": []
}, },
{ {
"name": "unique.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)", "(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)",
"(1, 3, 5, 4)", "(1, 3, 5, 4)",
"(1, 3, 5, 4)" "(1, 3, 5, 4)"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "unique.mq",
"stdin_lines": []
}, },
{ {
"name": "max.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"5", "5",
"209", "209",
"109", "109",
"10" "10"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "max.mq",
"stdin_lines": []
}, },
{ {
"name": "digits.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"(3, 4, 5, 6)", "(3, 4, 5, 6)",
"(1, 0)", "(1, 0)",
@ -215,73 +225,286 @@
"(0, 5)", "(0, 5)",
"(1, 2, 3, 4, 5, 6, 7, 8)" "(1, 2, 3, 4, 5, 6, 7, 8)"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "digits.mq",
"stdin_lines": []
}, },
{ {
"exit_code": 0,
"stdout_lines": [
"-4",
"-4",
"-4",
"-4",
"-5",
"4",
"5",
"5",
"5",
"5"
],
"stderr_lines": [],
"midi_events": null,
"name": "ceil.mq", "name": "ceil.mq",
"exit_code": 0, "stdin_lines": []
"stdin_lines": [],
"stdout_lines": [
"-4",
"-4",
"-4",
"-4",
"-5",
"4",
"5",
"5",
"5",
"5"
],
"stderr_lines": []
}, },
{ {
"exit_code": 0,
"stdout_lines": [
"-4",
"-5",
"-5",
"-5",
"-5",
"4",
"4",
"4",
"4",
"5"
],
"stderr_lines": [],
"midi_events": null,
"name": "floor.mq", "name": "floor.mq",
"stdin_lines": []
},
{
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"-4",
"-4",
"-4", "-4",
"-5", "-5",
"-5", "-5",
"-5",
"-5",
"4",
"4",
"4", "4",
"4", "4",
"5",
"5",
"5" "5"
], ],
"stderr_lines": [] "stderr_lines": [],
}, "midi_events": null,
{
"name": "round.mq", "name": "round.mq",
"exit_code": 0, "stdin_lines": []
"stdin_lines": [],
"stdout_lines": [
"-4",
"-4",
"-4",
"-5",
"-5",
"4",
"4",
"5",
"5",
"5"
],
"stderr_lines": []
}, },
{ {
"name": "duration.mq",
"exit_code": 0, "exit_code": 0,
"stdin_lines": [],
"stdout_lines": [ "stdout_lines": [
"3/4", "1/4",
"1/4", "1/4",
"1", "1",
"3/10" "3/10"
], ],
"stderr_lines": [] "stderr_lines": [],
"midi_events": null,
"name": "duration.mq",
"stdin_lines": []
}
]
},
{
"name": "music",
"cases": [
{
"exit_code": 0,
"stdout_lines": [],
"stderr_lines": [],
"midi_events": [
{
"type": "note_on",
"args": [
"0",
"60"
],
"time": 0.008109543001410202
},
{
"type": "note_off",
"args": [
"0",
"60"
],
"time": 0.07068762000017159
},
{
"type": "note_on",
"args": [
"0",
"61"
],
"time": 0.07074685500083433
},
{
"type": "note_off",
"args": [
"0",
"61"
],
"time": 0.13358810300087498
},
{
"type": "note_on",
"args": [
"0",
"62"
],
"time": 0.13376853900081187
},
{
"type": "note_off",
"args": [
"0",
"62"
],
"time": 0.19631889199990837
},
{
"type": "note_on",
"args": [
"0",
"63"
],
"time": 0.1964232610007457
},
{
"type": "note_off",
"args": [
"0",
"63"
],
"time": 0.2592293600009725
},
{
"type": "note_on",
"args": [
"0",
"64"
],
"time": 0.2593720190016029
},
{
"type": "note_off",
"args": [
"0",
"64"
],
"time": 0.321934201001568
},
{
"type": "note_on",
"args": [
"0",
"65"
],
"time": 0.3221377190002386
},
{
"type": "note_off",
"args": [
"0",
"65"
],
"time": 0.38442845700046746
},
{
"type": "note_on",
"args": [
"0",
"66"
],
"time": 0.3844692580005358
},
{
"type": "note_off",
"args": [
"0",
"66"
],
"time": 0.44720406100168475
},
{
"type": "note_on",
"args": [
"0",
"67"
],
"time": 0.44733363700106565
},
{
"type": "note_off",
"args": [
"0",
"67"
],
"time": 0.5101613840015489
},
{
"type": "note_on",
"args": [
"0",
"68"
],
"time": 0.5104183930016006
},
{
"type": "note_off",
"args": [
"0",
"68"
],
"time": 0.5728266430014628
},
{
"type": "note_on",
"args": [
"0",
"69"
],
"time": 0.5729434800014133
},
{
"type": "note_off",
"args": [
"0",
"69"
],
"time": 0.6357513590010058
},
{
"type": "note_on",
"args": [
"0",
"70"
],
"time": 0.6359626870016655
},
{
"type": "note_off",
"args": [
"0",
"70"
],
"time": 0.698417817000518
},
{
"type": "note_on",
"args": [
"0",
"71"
],
"time": 0.6985397290009132
},
{
"type": "note_off",
"args": [
"0",
"71"
],
"time": 0.7612005600003613
}
],
"name": "all_midi_notes.mq",
"stdin_lines": []
} }
] ]
} }

61
scripts/test.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import argparse import argparse
import dacite
import dataclasses import dataclasses
import json import json
import os import os
@ -19,6 +20,7 @@ except ModuleNotFoundError as e:
TEST_DIR = "regression-tests" TEST_DIR = "regression-tests"
TEST_DB = "test_db.json" TEST_DB = "test_db.json"
INTERPRETER = "bin/linux/debug/musique" INTERPRETER = "bin/linux/debug/musique"
MIDI_TOLERANCE = 0.005
@dataclasses.dataclass @dataclasses.dataclass
class MidiEvent: class MidiEvent:
@ -28,6 +30,7 @@ class MidiEvent:
def connect_to_default_midi_port(): def connect_to_default_midi_port():
global midi_client global midi_client
global port # this object keeps alive port, so it needs to live in global space (= live as long as program)
midi_client = alsa.SequencerClient('Musique Tester') midi_client = alsa.SequencerClient('Musique Tester')
ports = midi_client.list_ports(input = True) ports = midi_client.list_ports(input = True)
@ -49,7 +52,7 @@ def listen_for_midi_events() -> list[MidiEvent] | None:
events = [] events = []
while True: while True:
event = midi_client.event_input(timeout=2) event = midi_client.event_input(timeout=5)
if event is None: if event is None:
break break
end_time = time.monotonic() end_time = time.monotonic()
@ -68,6 +71,24 @@ def normalize_events(events) -> typing.Generator[MidiEvent, None, None]:
case _: case _:
assert False, f"Unmatched event type: {event.type}" assert False, f"Unmatched event type: {event.type}"
def compare_midi(xs: list[MidiEvent] | None, ys: list[MidiEvent] | None) -> bool:
if xs is None or ys is None:
return (xs is None) == (ys is None)
# TODO Can we get better performance then O(n^2) algorithm?
# Or at lexst optimize implementation of this one?
if len(xs) != len(ys):
return False
for x in xs:
if not any(x.type == y.type \
and x.args == y.args \
and x.time >= y.time - MIDI_TOLERANCE and x.time <= y.time + MIDI_TOLERANCE
for y in ys):
return False
return True
@dataclasses.dataclass @dataclasses.dataclass
class Result: class Result:
exit_code: int = 0 exit_code: int = 0
@ -80,7 +101,7 @@ class TestCase(Result):
name: str = "<unnamed>" name: str = "<unnamed>"
stdin_lines: list[str] = dataclasses.field(default_factory=list) stdin_lines: list[str] = dataclasses.field(default_factory=list)
def run(self, interpreter: str, source: str, cwd: str, capture_midi: bool = False) -> Result: def run(self, interpreter: str, source: str, cwd: str, *, capture_midi: bool = False) -> Result:
process = subprocess.Popen( process = subprocess.Popen(
args=[interpreter, source, "-q"], args=[interpreter, source, "-q"],
cwd=cwd, cwd=cwd,
@ -103,25 +124,36 @@ class TestCase(Result):
def record(self, interpreter: str, source: str, cwd: str): def record(self, interpreter: str, source: str, cwd: str):
print(f"Recording case {self.name}") print(f"Recording case {self.name}")
result = self.run(interpreter, source, cwd) result = self.run(interpreter, source, cwd, capture_midi=True)
changes = [] changes = []
if self.exit_code != result.exit_code: changes.append("exit code") if self.exit_code != result.exit_code: changes.append("exit code")
if self.stderr_lines != result.stderr_lines: changes.append("stderr") if self.stderr_lines != result.stderr_lines: changes.append("stderr")
if self.stdout_lines != result.stdout_lines: changes.append("stdout") if self.stdout_lines != result.stdout_lines: changes.append("stdout")
if self.midi_events != result.midi_events: changes.append("midi")
if changes: if changes:
print(f" changed: {', '.join(changes)}") print(f" changed: {', '.join(changes)}")
self.exit_code = result.exit_code self.exit_code = result.exit_code
self.stderr_lines = result.stderr_lines self.stderr_lines = result.stderr_lines
self.stdout_lines = result.stdout_lines self.stdout_lines = result.stdout_lines
self.midi_events = result.midi_events
def test(self, interpreter: str, source: str, cwd: str): def test(self, interpreter: str, source: str, cwd: str):
print(f" Testing case {self.name} ", end="") print(f" Testing case {self.name} ", end="")
result = self.run(interpreter, source, cwd)
capture_midi = self.midi_events is not None
result = self.run(interpreter, source, cwd, capture_midi=capture_midi)
if self.midi_events is not None:
midi_events = [dacite.from_dict(MidiEvent, event) for event in self.midi_events]
else:
midi_events = None
if self.exit_code == result.exit_code \ if self.exit_code == result.exit_code \
and self.stdout_lines == result.stdout_lines \ and self.stdout_lines == result.stdout_lines \
and self.stderr_lines == result.stderr_lines: and self.stderr_lines == result.stderr_lines \
and compare_midi(midi_events, result.midi_events):
print("ok") print("ok")
return True return True
@ -214,6 +246,25 @@ def update(path: str) -> list[tuple[TestSuite, TestCase]]:
print("Use --add to add new test case") print("Use --add to add new test case")
return [] return []
def testcase(path: str) -> list[tuple[TestSuite, TestCase]]:
test_suite, test_case = suite_case_from_path(path)
for suite in suites:
if suite.name == test_suite:
break
else:
print(f"Cannot test case {test_case} where suite {test_suite} was not defined yet.")
print("Use --add to add new test case")
return []
for case in suite.cases:
if case.name == test_case:
return [(suite, case)]
print(f"Case {test_case} doesn't exists in suite {test_suite}")
print("Use --add to add new test case")
return []
def traverse(discover: bool, update: bool) -> list[tuple[TestSuite, TestCase]]: def traverse(discover: bool, update: bool) -> list[tuple[TestSuite, TestCase]]:
to_record = list[tuple[TestSuite, TestCase]]() to_record = list[tuple[TestSuite, TestCase]]()
if discover: if discover: