Record and verify musical test cases

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- `port` function have proper semantics now
+
 ## [0.4.0]
 
 ### Added

musique/interpreter/builtin_functions.cc

@@ -1637,8 +1637,8 @@ static Result<Value> builtin_port(Interpreter &interpreter, std::vector<Value> a
 		for (auto const& [key, port] : Context::established_connections) {
 			if (port == interpreter.current_context->port) {
 				return std::visit(Overloaded {
-					[](midi::connections::Virtual_Port) { return Value(Symbol("virtual")); },
+					[](midi::Virtual_Port) { return Value(Symbol("virtual")); },
-					[](midi::connections::Established_Port port) { return Value(Number(port)); },
+					[](midi::Established_Port port) { return Value(Number(port)); },
 				}, key);
 			}
 		}
@@ -1647,14 +1647,14 @@ static Result<Value> builtin_port(Interpreter &interpreter, std::vector<Value> a
 
 	if (auto a = match<Number>(args)) {
 		auto [port_number] = *a;
-		Try(interpreter.current_context->connect(port_number.floor().as_int()));
+		Try(interpreter.current_context->connect(midi::Established_Port(port_number.floor().as_int())));
 		return {};
 	}
 
 	if (auto a = match<Symbol>(args)) {
 		auto [port_type] = *a;
 		if (port_type == "virtual") {
-			Try(interpreter.current_context->connect(std::nullopt));
+			Try(interpreter.current_context->connect(midi::Virtual_Port{}));
 			return {};
 		}
 

musique/interpreter/context.cc

@@ -14,11 +14,11 @@ std::chrono::duration<float> Context::length_to_duration(std::optional<Number> l
 }
 
 template<>
-struct std::hash<midi::connections::Key>
+struct std::hash<midi::Port>
 {
-	std::size_t operator()(midi::connections::Key const& value) const
+	std::size_t operator()(midi::Port const& value) const
 	{
-		using namespace midi::connections;
+		using namespace midi;
 		return hash_combine(value.index(), std::visit(Overloaded {
 			[](Virtual_Port) { return 0u; },
 			[](Established_Port port) { return port; },
@@ -26,35 +26,33 @@ struct std::hash<midi::connections::Key>
 	}
 };
 
-std::unordered_map<midi::connections::Key, std::shared_ptr<midi::Connection>> Context::established_connections;
+std::unordered_map<midi::Port, std::shared_ptr<midi::Connection>> Context::established_connections;
 
 /// Establish connection to given port
-std::optional<Error> Context::connect(std::optional<Port_Number> port_number)
+std::optional<Error> Context::connect(std::optional<midi::Port> desired_port)
 {
 	// FIXME This function doesn't support creating virtual ports when established ports are available
-	using namespace midi::connections;
+	if (!desired_port) {
-	auto const key = port_number ? Key(*port_number) : Key(Virtual_Port{});
+		if (not established_connections.empty()) {
+			port = established_connections.begin()->second;
+			return std::nullopt;
+		}
+		auto connection = std::make_shared<midi::Rt_Midi>();
+		established_connections[connection->establish_any_connection()] = connection;
+		port = connection;
+		return std::nullopt;
+	}
 
-	if (auto it = established_connections.find(key); it != established_connections.end()) {
+	if (auto it = established_connections.find(*desired_port); it != established_connections.end()) {
 		port = it->second;
 		return std::nullopt;
 	}
 
-	if (port_number) {
 	auto connection = std::make_shared<midi::Rt_Midi>();
-		connection->connect_output(*port_number);
+	connection->connect(*desired_port);
-		established_connections[*port_number] = connection;
+	established_connections[*desired_port] = connection;
-		port = connection;
-		return std::nullopt;
-	}
-
-	auto connection = std::make_shared<midi::Rt_Midi>();
-	if (connection->connect_or_create_output()) {
-		established_connections[0u] = connection;
-	} else {
-		established_connections[Virtual_Port{}] = connection;
-	}
 	port = connection;
+
 	return std::nullopt;
 }
 

musique/interpreter/context.hh

@@ -9,17 +9,6 @@
 #include <musique/value/note.hh>
 #include <musique/value/number.hh>
 
-namespace midi::connections
-{
-	using Established_Port = unsigned int;
-
-	struct Virtual_Port
-	{
-		bool operator==(Virtual_Port const&) const = default;
-	};
-
-	using Key = std::variant<Established_Port, Virtual_Port>;
-}
 
 /// Context holds default values for music related actions
 struct Context
@@ -39,12 +28,12 @@ struct Context
 	using Port_Number = unsigned int;
 
 	/// Connections that have been established so far
-	static std::unordered_map<midi::connections::Key, std::shared_ptr<midi::Connection>> established_connections;
+	static std::unordered_map<midi::Port, std::shared_ptr<midi::Connection>> established_connections;
 
 	/// Establish connection to given port
 	///
 	/// If port number wasn't provided connect to first existing one or create one
-	std::optional<Error> connect(std::optional<Port_Number>);
+	std::optional<Error> connect(std::optional<midi::Port>);
 
 	/// Fills empty places in Note like octave and length with default values from context
 	Note fill(Note) const;

musique/midi/midi.hh

@@ -4,10 +4,20 @@
 #include <functional>
 #include <optional>
 #include <string>
+#include <variant>
 
 // Documentation of midi messages available at http://midi.teragonaudio.com/tech/midispec.htm
 namespace midi
 {
+	using Established_Port = unsigned int;
+
+	struct Virtual_Port
+	{
+		bool operator==(Virtual_Port const&) const = default;
+	};
+
+	using Port = std::variant<Established_Port, Virtual_Port>;
+
 	struct Connection
 	{
 		virtual ~Connection() = default;
@@ -26,18 +36,15 @@ namespace midi
 	{
 		~Rt_Midi() override = default;
 
-		bool connect_or_create_output();
+		/// Connect to existing MIDI output or create virtual port
+		Port establish_any_connection();
 
-		/// Connect with MIDI virtual port
+		/// Connect to given port
-		void connect_output();
+		void connect(Port port);
-
-		/// Connect with specific MIDI port for outputing MIDI messages
-		void connect_output(unsigned target);
 
 		/// List available ports
 		void list_ports(std::ostream &out) const;
 
-
 		bool supports_output() const override;
 
 		void send_note_on (uint8_t channel, uint8_t note_number, uint8_t velocity) override;
@@ -45,6 +52,7 @@ namespace midi
 		void send_program_change(uint8_t channel, uint8_t program) override;
 		void send_controller_change(uint8_t channel, uint8_t controller_number, uint8_t value) override;
 
+
 		std::optional<RtMidiOut> output;
 	};
 

musique/midi/rt_midi.cc

@@ -34,15 +34,16 @@ try {
 	std::exit(33);
 }
 
-bool midi::Rt_Midi::connect_or_create_output()
+midi::Port midi::Rt_Midi::establish_any_connection()
 try {
 	output.emplace();
 	if (output->getPortCount()) {
-		output->openPort(0);
+		output->openPort(0, "Musique");
-		return true;
+		return Established_Port { 0 };
 	}
+
 	output->openVirtualPort("Musique");
-	return false;
+	return Virtual_Port{};
 }
 catch (RtMidiError &error) {
 	// TODO(error)
@@ -50,23 +51,22 @@ catch (RtMidiError &error) {
 	std::exit(33);
 }
 
-void midi::Rt_Midi::connect_output()
+void midi::Rt_Midi::connect(midi::Port port)
 try {
-	ensure(not output.has_value(), "Reconeccting is not supported yet");
+	std::visit(Overloaded{
+		[this](midi::Virtual_Port) {
 			output.emplace();
 			output->openVirtualPort("Musique");
-} catch (RtMidiError &error) {
+		},
-	// TODO(error)
+		[this](midi::Established_Port port) {
-	std::cerr << "Failed to use MIDI connection: " << error.getMessage() << std::endl;
-	std::exit(33);
-}
-
-void midi::Rt_Midi::connect_output(unsigned target)
-try {
-	ensure(not output.has_value(), "Reconeccting is not supported yet");
 			output.emplace();
-	output->openPort(target);
+			output->openPort(port, "Musique");
-} catch (RtMidiError &error) {
+		},
+	}, port);
+	output->setClientName("Musique");
+	output->setPortName("Musique");
+}
+catch (RtMidiError &error) {
 	// TODO(error)
 	std::cerr << "Failed to use MIDI connection: " << error.getMessage() << std::endl;
 	std::exit(33);

regression-tests/music/all_midi_notes.mq

@@ -0,0 +1,2 @@
+len tn,
+play (c4 + up 12)

regression-tests/test_db.json

@@ -3,9 +3,7 @@
     "name": "boolean",
     "cases": [
       {
-        "name": "logical_or.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "false",
           "true",
@@ -18,12 +16,13 @@
           "10",
           "42"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "logical_or.mq",
+        "stdin_lines": []
       },
       {
-        "name": "logical_and.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "false",
           "false",
@@ -37,7 +36,10 @@
           "32",
           "42"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "logical_and.mq",
+        "stdin_lines": []
       }
     ]
   },
@@ -45,9 +47,7 @@
     "name": "builtin",
     "cases": [
       {
-        "name": "permute.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "(0, 1, 3, 2)",
           "(0, 2, 1, 3)",
@@ -77,12 +77,13 @@
           "(0, 1, 4, (3, 2))",
           "(0, 4, (3, 2), 1)"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "permute.mq",
+        "stdin_lines": []
       },
       {
-        "name": "range.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "()",
           "()",
@@ -100,35 +101,38 @@
           "(9, 8, 7, 6, 5, 4, 3, 2, 1)",
           "(9, 7, 5, 3, 1)"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "range.mq",
+        "stdin_lines": []
       },
       {
-        "name": "min.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "1",
           "200",
           "100",
           "0"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "min.mq",
+        "stdin_lines": []
       },
       {
-        "name": "call.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "42",
           "11",
           "43"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "call.mq",
+        "stdin_lines": []
       },
       {
-        "name": "if.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "1",
           "2",
@@ -138,35 +142,38 @@
           "200",
           "9"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "if.mq",
+        "stdin_lines": []
       },
       {
-        "name": "uniq.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)",
           "(1, 3, 5, 3, 4, 1)",
           "(1, 3, 5, 3, 4, 1)"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "uniq.mq",
+        "stdin_lines": []
       },
       {
-        "name": "reverse.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "()",
           "(9, 8, 7, 6, 5, 4, 3, 2, 1, 0)",
           "(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)",
           "(9, 8, 7, 6, 5, 4, (1, 2, 3))"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "reverse.mq",
+        "stdin_lines": []
       },
       {
-        "name": "typeof.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "array",
           "number",
@@ -176,35 +183,38 @@
           "nil",
           "intrinsic"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "typeof.mq",
+        "stdin_lines": []
       },
       {
-        "name": "unique.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)",
           "(1, 3, 5, 4)",
           "(1, 3, 5, 4)"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "unique.mq",
+        "stdin_lines": []
       },
       {
-        "name": "max.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "5",
           "209",
           "109",
           "10"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "max.mq",
+        "stdin_lines": []
       },
       {
-        "name": "digits.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
           "(3, 4, 5, 6)",
           "(1, 0)",
@@ -215,73 +225,286 @@
           "(0, 5)",
           "(1, 2, 3, 4, 5, 6, 7, 8)"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "digits.mq",
+        "stdin_lines": []
       },
       {
+        "exit_code": 0,
+        "stdout_lines": [
+          "-4",
+          "-4",
+          "-4",
+          "-4",
+          "-5",
+          "4",
+          "5",
+          "5",
+          "5",
+          "5"
+        ],
+        "stderr_lines": [],
+        "midi_events": null,
         "name": "ceil.mq",
-        "exit_code": 0,
+        "stdin_lines": []
-        "stdin_lines": [],
-        "stdout_lines": [
-          "-4",
-          "-4",
-          "-4",
-          "-4",
-          "-5",
-          "4",
-          "5",
-          "5",
-          "5",
-          "5"
-        ],
-        "stderr_lines": []
       },
       {
+        "exit_code": 0,
+        "stdout_lines": [
+          "-4",
+          "-5",
+          "-5",
+          "-5",
+          "-5",
+          "4",
+          "4",
+          "4",
+          "4",
+          "5"
+        ],
+        "stderr_lines": [],
+        "midi_events": null,
         "name": "floor.mq",
+        "stdin_lines": []
+      },
+      {
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
+          "-4",
+          "-4",
           "-4",
           "-5",
           "-5",
-          "-5",
-          "-5",
-          "4",
-          "4",
           "4",
           "4",
+          "5",
+          "5",
           "5"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
-      },
+        "midi_events": null,
-      {
         "name": "round.mq",
-        "exit_code": 0,
+        "stdin_lines": []
-        "stdin_lines": [],
-        "stdout_lines": [
-          "-4",
-          "-4",
-          "-4",
-          "-5",
-          "-5",
-          "4",
-          "4",
-          "5",
-          "5",
-          "5"
-        ],
-        "stderr_lines": []
       },
       {
-        "name": "duration.mq",
         "exit_code": 0,
-        "stdin_lines": [],
         "stdout_lines": [
-          "3/4",
+          "1/4",
           "1/4",
           "1",
           "3/10"
         ],
-        "stderr_lines": []
+        "stderr_lines": [],
+        "midi_events": null,
+        "name": "duration.mq",
+        "stdin_lines": []
+      }
+    ]
+  },
+  {
+    "name": "music",
+    "cases": [
+      {
+        "exit_code": 0,
+        "stdout_lines": [],
+        "stderr_lines": [],
+        "midi_events": [
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "60"
+            ],
+            "time": 0.008109543001410202
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "60"
+            ],
+            "time": 0.07068762000017159
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "61"
+            ],
+            "time": 0.07074685500083433
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "61"
+            ],
+            "time": 0.13358810300087498
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "62"
+            ],
+            "time": 0.13376853900081187
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "62"
+            ],
+            "time": 0.19631889199990837
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "63"
+            ],
+            "time": 0.1964232610007457
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "63"
+            ],
+            "time": 0.2592293600009725
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "64"
+            ],
+            "time": 0.2593720190016029
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "64"
+            ],
+            "time": 0.321934201001568
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "65"
+            ],
+            "time": 0.3221377190002386
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "65"
+            ],
+            "time": 0.38442845700046746
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "66"
+            ],
+            "time": 0.3844692580005358
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "66"
+            ],
+            "time": 0.44720406100168475
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "67"
+            ],
+            "time": 0.44733363700106565
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "67"
+            ],
+            "time": 0.5101613840015489
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "68"
+            ],
+            "time": 0.5104183930016006
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "68"
+            ],
+            "time": 0.5728266430014628
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "69"
+            ],
+            "time": 0.5729434800014133
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "69"
+            ],
+            "time": 0.6357513590010058
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "70"
+            ],
+            "time": 0.6359626870016655
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "70"
+            ],
+            "time": 0.698417817000518
+          },
+          {
+            "type": "note_on",
+            "args": [
+              "0",
+              "71"
+            ],
+            "time": 0.6985397290009132
+          },
+          {
+            "type": "note_off",
+            "args": [
+              "0",
+              "71"
+            ],
+            "time": 0.7612005600003613
+          }
+        ],
+        "name": "all_midi_notes.mq",
+        "stdin_lines": []
       }
     ]
   }

scripts/test.py

@@ -1,59 +1,159 @@
 #!/usr/bin/env python3
 import argparse
+import dacite
 import dataclasses
 import json
 import os
 import subprocess
+import time
+import typing
+
+try:
+    import alsa_midi as alsa
+    WITH_MIDI = True
+except ModuleNotFoundError as e:
+    print(f"ERROR: Failed to load module alsa_midi: {e}")
+    print("        Tests using this module will automatically fail\n"
+          "        if not provided with --skip-midi flag")
+    exit(1)
 
 TEST_DIR = "regression-tests"
 TEST_DB = "test_db.json"
 INTERPRETER = "bin/linux/debug/musique"
+MIDI_TOLERANCE = 0.005
+
+@dataclasses.dataclass
+class MidiEvent:
+    type: str
+    args: list[str]
+    time: float
+
+def connect_to_default_midi_port():
+    global midi_client
+    global port # this object keeps alive port, so it needs to live in global space (= live as long as program)
+    midi_client = alsa.SequencerClient('Musique Tester')
+    ports = midi_client.list_ports(input = True)
+
+    for p in ports:
+        if p.client_id == 14:
+            input_port = p
+            break
+    else:
+        assert False, "Linux default MIDI port not found"
+
+    port = midi_client.create_port('Musique tester')
+    port.connect_from(input_port)
+
+def listen_for_midi_events() -> list[MidiEvent] | None:
+    if not WITH_MIDI:
+        return None
+
+    zero_time = time.monotonic()
+
+    events = []
+    while True:
+        event = midi_client.event_input(timeout=5)
+        if event is None:
+            break
+        end_time = time.monotonic()
+        events.append((event, end_time - zero_time))
+    return events
+
+
+def normalize_events(events) -> typing.Generator[MidiEvent, None, None]:
+    for event, time in events:
+        match event:
+            case alsa.event.NoteOnEvent():
+                # TODO Support velocity
+                yield MidiEvent(type='note_on', args=[str(event.channel), str(event.note)], time=time)
+            case alsa.event.NoteOffEvent():
+                yield MidiEvent(type='note_off', args=[str(event.channel), str(event.note)], time=time)
+            case _:
+                assert False, f"Unmatched event type: {event.type}"
+
+def compare_midi(xs: list[MidiEvent] | None, ys: list[MidiEvent] | None) -> bool:
+    if xs is None or ys is None:
+        return (xs is None) == (ys is None)
+
+    # TODO Can we get better performance then O(n^2) algorithm?
+    #      Or at lexst optimize implementation of this one?
+    if len(xs) != len(ys):
+        return False
+
+    for x in xs:
+        if not any(x.type == y.type \
+                and x.args == y.args \
+                and x.time >= y.time - MIDI_TOLERANCE and x.time <= y.time + MIDI_TOLERANCE
+               for y in ys):
+            return False
+    return True
+
 
 @dataclasses.dataclass
 class Result:
     exit_code:    int       = 0
-    stdin_lines:  list[str] = dataclasses.field(default_factory=list)
     stdout_lines: list[str] = dataclasses.field(default_factory=list)
     stderr_lines: list[str] = dataclasses.field(default_factory=list)
+    midi_events:  list[MidiEvent] | None = None
 
 @dataclasses.dataclass
-class TestCase:
+class TestCase(Result):
-    name:         str
+    name: str = "<unnamed>"
-    exit_code:    int       = 0
     stdin_lines:  list[str] = dataclasses.field(default_factory=list)
-    stdout_lines: list[str] = dataclasses.field(default_factory=list)
-    stderr_lines: list[str] = dataclasses.field(default_factory=list)
 
-    def run(self, interpreter: str, source: str, cwd: str):
+    def run(self, interpreter: str, source: str, cwd: str, *, capture_midi: bool = False) -> Result:
-        result = subprocess.run(
+        process = subprocess.Popen(
             args=[interpreter, source, "-q"],
-            capture_output=True,
             cwd=cwd,
-            text=True
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            stdin=subprocess.PIPE
         )
+
+        midi_events = listen_for_midi_events() if capture_midi else None
+        stdout, stderr = process.communicate()
+
         return Result(
-            exit_code=result.returncode,
+            exit_code=process.wait(),
-            stdout_lines=result.stdout.splitlines(keepends=False),
+            stdout_lines=stdout.splitlines(keepends=False),
-            stderr_lines=result.stderr.splitlines(keepends=False)
+            stderr_lines=stderr.splitlines(keepends=False),
+            midi_events = None if midi_events is None else \
+                    list(normalize_events(midi_events))
         )
 
     def record(self, interpreter: str, source: str, cwd: str):
         print(f"Recording case {self.name}")
-        result = self.run(interpreter, source, cwd)
+        result = self.run(interpreter, source, cwd, capture_midi=True)
 
         changes = []
         if self.exit_code    != result.exit_code:    changes.append("exit code")
         if self.stderr_lines != result.stderr_lines: changes.append("stderr")
         if self.stdout_lines != result.stdout_lines: changes.append("stdout")
+        if self.midi_events  != result.midi_events:  changes.append("midi")
         if changes:
             print(f"  changed: {', '.join(changes)}")
+            self.exit_code    = result.exit_code
+            self.stderr_lines = result.stderr_lines
+            self.stdout_lines = result.stdout_lines
+            self.midi_events  = result.midi_events
 
-        self.exit_code, self.stderr_lines, self.stdout_lines = result.exit_code, result.stderr_lines, result.stdout_lines
 
     def test(self, interpreter: str, source: str, cwd: str):
         print(f"  Testing case {self.name}  ", end="")
-        result = self.run(interpreter, source, cwd)
+
-        if self.exit_code == result.exit_code and self.stdout_lines == result.stdout_lines and self.stderr_lines == result.stderr_lines:
+        capture_midi = self.midi_events is not None
+
+        result = self.run(interpreter, source, cwd, capture_midi=capture_midi)
+        if self.midi_events is not None:
+            midi_events = [dacite.from_dict(MidiEvent, event) for event in self.midi_events]
+        else:
+            midi_events = None
+
+        if self.exit_code == result.exit_code \
+            and self.stdout_lines == result.stdout_lines \
+            and self.stderr_lines == result.stderr_lines \
+            and compare_midi(midi_events, result.midi_events):
             print("ok")
             return True
 
@@ -146,6 +246,25 @@ def update(path: str) -> list[tuple[TestSuite, TestCase]]:
     print("Use --add to add new test case")
     return []
 
+def testcase(path: str) -> list[tuple[TestSuite, TestCase]]:
+    test_suite, test_case = suite_case_from_path(path)
+
+    for suite in suites:
+        if suite.name == test_suite:
+            break
+    else:
+        print(f"Cannot test case {test_case} where suite {test_suite} was not defined yet.")
+        print("Use --add to add new test case")
+        return []
+
+    for case in suite.cases:
+        if case.name == test_case:
+            return [(suite, case)]
+
+    print(f"Case {test_case} doesn't exists in suite {test_suite}")
+    print("Use --add to add new test case")
+    return []
+
 def traverse(discover: bool, update: bool) -> list[tuple[TestSuite, TestCase]]:
     to_record = list[tuple[TestSuite, TestCase]]()
     if discover:
@@ -191,9 +310,14 @@ if __name__ == "__main__":
     parser.add_argument("--update-all", action="store_true", help="Update all tests", dest="update_all")
     parser.add_argument("-a", "--add", action="append", help="Add new test to test suite", default=[])
     parser.add_argument("-u", "--update", action="append", help="Update test case", default=[])
+    parser.add_argument('--skip-midi', action="store_true", help="Skip tests expecting MIDI communication", default=False, dest="skip_midi")
 
     args = parser.parse_args()
 
+    WITH_MIDI = WITH_MIDI and not args.skip_midi
+    if WITH_MIDI:
+        connect_to_default_midi_port()
+
     root = os.path.dirname(os.path.dirname(__file__))
     testing_dir = os.path.join(root, TEST_DIR)
     test_db_path = os.path.join(testing_dir, TEST_DB)