s416496/musique
1
0
Fork 0
Code Issues 3 Pull Requests 1 Projects Releases Wiki Activity

Record and verify musical test cases

Browse Source
This commit is contained in:
Robert Bendun 2023-02-15 01:18:35 +01:00
parent 08cc14e50b
commit cf83432b15
4 changed files with 363 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
musique/main.cc
View File

@ -169,7 +169,7 @@ struct Runner
ensure(the == nullptr, "Only one instance of runner is supported");
the = this;
// interpreter.current_context->connect(std::nullopt);
interpreter.current_context->connect(std::nullopt);
Env::global->force_define("say", +[](Interpreter &interpreter, std::vector<Value> args) -> Result<Value> {
for (auto it = args.begin(); it != args.end(); ++it) {

2
regression-tests/music/all_midi_notes.mq Normal file
View File

@ -0,0 +1,2 @@
len tn,
play (c4 + up 12)

385
regression-tests/test_db.json
View File

@ -3,9 +3,7 @@
"name": "boolean",
"cases": [
{
"name": "logical_or.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"false",
"true",
@ -18,12 +16,13 @@
"10",
"42"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "logical_or.mq",
"stdin_lines": []
},
{
"name": "logical_and.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"false",
"false",
@ -37,7 +36,10 @@
"32",
"42"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "logical_and.mq",
"stdin_lines": []
}
]
},
@ -45,9 +47,7 @@
"name": "builtin",
"cases": [
{
"name": "permute.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"(0, 1, 3, 2)",
"(0, 2, 1, 3)",
@ -77,12 +77,13 @@
"(0, 1, 4, (3, 2))",
"(0, 4, (3, 2), 1)"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "permute.mq",
"stdin_lines": []
},
{
"name": "range.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"()",
"()",
@ -100,35 +101,38 @@
"(9, 8, 7, 6, 5, 4, 3, 2, 1)",
"(9, 7, 5, 3, 1)"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "range.mq",
"stdin_lines": []
},
{
"name": "min.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"1",
"200",
"100",
"0"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "min.mq",
"stdin_lines": []
},
{
"name": "call.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"42",
"11",
"43"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "call.mq",
"stdin_lines": []
},
{
"name": "if.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"1",
"2",
@ -138,35 +142,38 @@
"200",
"9"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "if.mq",
"stdin_lines": []
},
{
"name": "uniq.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)",
"(1, 3, 5, 3, 4, 1)",
"(1, 3, 5, 3, 4, 1)"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "uniq.mq",
"stdin_lines": []
},
{
"name": "reverse.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"()",
"(9, 8, 7, 6, 5, 4, 3, 2, 1, 0)",
"(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)",
"(9, 8, 7, 6, 5, 4, (1, 2, 3))"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "reverse.mq",
"stdin_lines": []
},
{
"name": "typeof.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"array",
"number",
@ -176,35 +183,38 @@
"nil",
"intrinsic"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "typeof.mq",
"stdin_lines": []
},
{
"name": "unique.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)",
"(1, 3, 5, 4)",
"(1, 3, 5, 4)"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "unique.mq",
"stdin_lines": []
},
{
"name": "max.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"5",
"209",
"109",
"10"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "max.mq",
"stdin_lines": []
},
{
"name": "digits.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"(3, 4, 5, 6)",
"(1, 0)",
@ -215,73 +225,286 @@
"(0, 5)",
"(1, 2, 3, 4, 5, 6, 7, 8)"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "digits.mq",
"stdin_lines": []
},
{
"exit_code": 0,
"stdout_lines": [
"-4",
"-4",
"-4",
"-4",
"-5",
"4",
"5",
"5",
"5",
"5"
],
"stderr_lines": [],
"midi_events": null,
"name": "ceil.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"-4",
"-4",
"-4",
"-4",
"-5",
"4",
"5",
"5",
"5",
"5"
],
"stderr_lines": []
"stdin_lines": []
},
{
"exit_code": 0,
"stdout_lines": [
"-4",
"-5",
"-5",
"-5",
"-5",
"4",
"4",
"4",
"4",
"5"
],
"stderr_lines": [],
"midi_events": null,
"name": "floor.mq",
"stdin_lines": []
},
{
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"-4",
"-4",
"-4",
"-5",
"-5",
"-5",
"-5",
"4",
"4",
"4",
"4",
"5",
"5",
"5"
],
"stderr_lines": []
},
{
"stderr_lines": [],
"midi_events": null,
"name": "round.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"-4",
"-4",
"-4",
"-5",
"-5",
"4",
"4",
"5",
"5",
"5"
],
"stderr_lines": []
"stdin_lines": []
},
{
"name": "duration.mq",
"exit_code": 0,
"stdin_lines": [],
"stdout_lines": [
"3/4",
"1/4",
"1/4",
"1",
"3/10"
],
"stderr_lines": []
"stderr_lines": [],
"midi_events": null,
"name": "duration.mq",
"stdin_lines": []
}
]
},
{
"name": "music",
"cases": [
{
"exit_code": 0,
"stdout_lines": [],
"stderr_lines": [],
"midi_events": [
{
"type": "note_on",
"args": [
"0",
"60"
],
"time": 0.008109543001410202
},
{
"type": "note_off",
"args": [
"0",
"60"
],
"time": 0.07068762000017159
},
{
"type": "note_on",
"args": [
"0",
"61"
],
"time": 0.07074685500083433
},
{
"type": "note_off",
"args": [
"0",
"61"
],
"time": 0.13358810300087498
},
{
"type": "note_on",
"args": [
"0",
"62"
],
"time": 0.13376853900081187
},
{
"type": "note_off",
"args": [
"0",
"62"
],
"time": 0.19631889199990837
},
{
"type": "note_on",
"args": [
"0",
"63"
],
"time": 0.1964232610007457
},
{
"type": "note_off",
"args": [
"0",
"63"
],
"time": 0.2592293600009725
},
{
"type": "note_on",
"args": [
"0",
"64"
],
"time": 0.2593720190016029
},
{
"type": "note_off",
"args": [
"0",
"64"
],
"time": 0.321934201001568
},
{
"type": "note_on",
"args": [
"0",
"65"
],
"time": 0.3221377190002386
},
{
"type": "note_off",
"args": [
"0",
"65"
],
"time": 0.38442845700046746
},
{
"type": "note_on",
"args": [
"0",
"66"
],
"time": 0.3844692580005358
},
{
"type": "note_off",
"args": [
"0",
"66"
],
"time": 0.44720406100168475
},
{
"type": "note_on",
"args": [
"0",
"67"
],
"time": 0.44733363700106565
},
{
"type": "note_off",
"args": [
"0",
"67"
],
"time": 0.5101613840015489
},
{
"type": "note_on",
"args": [
"0",
"68"
],
"time": 0.5104183930016006
},
{
"type": "note_off",
"args": [
"0",
"68"
],
"time": 0.5728266430014628
},
{
"type": "note_on",
"args": [
"0",
"69"
],
"time": 0.5729434800014133
},
{
"type": "note_off",
"args": [
"0",
"69"
],
"time": 0.6357513590010058
},
{
"type": "note_on",
"args": [
"0",
"70"
],
"time": 0.6359626870016655
},
{
"type": "note_off",
"args": [
"0",
"70"
],
"time": 0.698417817000518
},
{
"type": "note_on",
"args": [
"0",
"71"
],
"time": 0.6985397290009132
},
{
"type": "note_off",
"args": [
"0",
"71"
],
"time": 0.7612005600003613
}
],
"name": "all_midi_notes.mq",
"stdin_lines": []
}
]
}

61
scripts/test.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python3
import argparse
import dacite
import dataclasses
import json
import os
@ -19,6 +20,7 @@ except ModuleNotFoundError as e:
TEST_DIR = "regression-tests"
TEST_DB = "test_db.json"
INTERPRETER = "bin/linux/debug/musique"
MIDI_TOLERANCE = 0.005
@dataclasses.dataclass
class MidiEvent:
@ -28,6 +30,7 @@ class MidiEvent:
def connect_to_default_midi_port():
global midi_client
global port # this object keeps alive port, so it needs to live in global space (= live as long as program)
midi_client = alsa.SequencerClient('Musique Tester')
ports = midi_client.list_ports(input = True)
@ -49,7 +52,7 @@ def listen_for_midi_events() -> list[MidiEvent] | None:
events = []
while True:
event = midi_client.event_input(timeout=2)
event = midi_client.event_input(timeout=5)
if event is None:
break
end_time = time.monotonic()
@ -68,6 +71,24 @@ def normalize_events(events) -> typing.Generator[MidiEvent, None, None]:
case _:
assert False, f"Unmatched event type: {event.type}"
def compare_midi(xs: list[MidiEvent] | None, ys: list[MidiEvent] | None) -> bool:
if xs is None or ys is None:
return (xs is None) == (ys is None)
# TODO Can we get better performance then O(n^2) algorithm?
# Or at lexst optimize implementation of this one?
if len(xs) != len(ys):
return False
for x in xs:
if not any(x.type == y.type \
and x.args == y.args \
and x.time >= y.time - MIDI_TOLERANCE and x.time <= y.time + MIDI_TOLERANCE
for y in ys):
return False
return True
@dataclasses.dataclass
class Result:
exit_code: int = 0
@ -80,7 +101,7 @@ class TestCase(Result):
name: str = "<unnamed>"
stdin_lines: list[str] = dataclasses.field(default_factory=list)
def run(self, interpreter: str, source: str, cwd: str, capture_midi: bool = False) -> Result:
def run(self, interpreter: str, source: str, cwd: str, *, capture_midi: bool = False) -> Result:
process = subprocess.Popen(
args=[interpreter, source, "-q"],
cwd=cwd,
@ -103,25 +124,36 @@ class TestCase(Result):
def record(self, interpreter: str, source: str, cwd: str):
print(f"Recording case {self.name}")
result = self.run(interpreter, source, cwd)
result = self.run(interpreter, source, cwd, capture_midi=True)
changes = []
if self.exit_code != result.exit_code: changes.append("exit code")
if self.stderr_lines != result.stderr_lines: changes.append("stderr")
if self.stdout_lines != result.stdout_lines: changes.append("stdout")
if self.midi_events != result.midi_events: changes.append("midi")
if changes:
print(f" changed: {', '.join(changes)}")
self.exit_code = result.exit_code
self.stderr_lines = result.stderr_lines
self.stdout_lines = result.stdout_lines
self.midi_events = result.midi_events
def test(self, interpreter: str, source: str, cwd: str):
print(f" Testing case {self.name} ", end="")
result = self.run(interpreter, source, cwd)
capture_midi = self.midi_events is not None
result = self.run(interpreter, source, cwd, capture_midi=capture_midi)
if self.midi_events is not None:
midi_events = [dacite.from_dict(MidiEvent, event) for event in self.midi_events]
else:
midi_events = None
if self.exit_code == result.exit_code \
and self.stdout_lines == result.stdout_lines \
and self.stderr_lines == result.stderr_lines:
and self.stderr_lines == result.stderr_lines \
and compare_midi(midi_events, result.midi_events):
print("ok")
return True
@ -214,6 +246,25 @@ def update(path: str) -> list[tuple[TestSuite, TestCase]]:
print("Use --add to add new test case")
return []
def testcase(path: str) -> list[tuple[TestSuite, TestCase]]:
test_suite, test_case = suite_case_from_path(path)
for suite in suites:
if suite.name == test_suite:
break
else:
print(f"Cannot test case {test_case} where suite {test_suite} was not defined yet.")
print("Use --add to add new test case")
return []
for case in suite.cases:
if case.name == test_case:
return [(suite, case)]
print(f"Case {test_case} doesn't exists in suite {test_suite}")
print("Use --add to add new test case")
return []
def traverse(discover: bool, update: bool) -> list[tuple[TestSuite, TestCase]]:
to_record = list[tuple[TestSuite, TestCase]]()
if discover: