From 0075ee208ff3c5da9e2df5965a78a368546808a6 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Tue, 29 Sep 2026 14:00:02 +1300 Subject: [PATCH] qet-mcp: exact-answer tests for qet_check, qet_continuity and the .elmt tools A mutation audit of the functions no audit had covered (tests needing no QElectroTech): these 13 caught 153 of 337 planted bugs (45 %). With these tests, 322 (96 %). qet_check was at 26/59 and qet_continuity at 24/37 even with the binary tests, since those look at a finding or two. - qet_check / qet_continuity: _run_qet replaced by a stub returning chosen log lines, so the whole answer is compared -- summary counts, "ok", passed, check_failures, each finding's count, note and sampled rows, sorting by severity, folio_number, the launch hint carried through, the folio argument in the script, folio bounds, and lines that only look like ours. - qet_element_build / qet_element_info: the written header, names, kind information and terminals, and the reported result, key for key; the refusals; geometry worked out by hand; every part kind's extent and written attributes; number formatting. - qet_element_search and its index: the index entry key for key, the cache, ranking (exact name, then first word, then length), the default and given limits. The 15 left are equivalent: timeouts and output limits, a ranking constant that only has to exceed 0, and a containment check the 5-unit margin keeps from ever failing. Tests only; 282/282. Co-Authored-By: Claude Opus 5.5 --- misc/qet-mcp/test_qet_mcp.py | 386 +++++++++++++++++++++++++++++++++++ 1 file changed, 386 insertions(+) diff --git a/misc/qet-mcp/test_qet_mcp.py b/misc/qet-mcp/test_qet_mcp.py index 24c1e2718..97d0a812f 100644 --- a/misc/qet-mcp/test_qet_mcp.py +++ b/misc/qet-mcp/test_qet_mcp.py @@ -30,6 +30,7 @@ import subprocess import sys import tempfile import unittest +from unittest import mock import xml.etree.ElementTree as ET from pathlib import Path @@ -709,6 +710,391 @@ class ElementBuild(unittest.TestCase): self.assertEqual(r["verified"]["names"]["en"], 'Coil "A" & ') +class ElementFileExact(unittest.TestCase): + """Exact answers for the tools that write and read .elmt files and need + no QElectroTech. Found by a mutation audit: the existing tests checked a + few fields, so dropping any other one from a result, or getting a + header number off by one, went unnoticed.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + m._ELEMENT_INDEX.clear() + + def tearDown(self): + self.tmp.cleanup() + + def test_fmt(self): + self.assertEqual([m._fmt(v) for v in (True, False, 3, 2.0, 2.5, -0.25, "x")], + ["true", "false", "3", "2", "2.5", "-0.25", "x"]) + + def test_part_extent_per_kind(self): + ext = m._part_extent + self.assertEqual(ext("line", {"x1": 1, "y1": 2, "x2": 3, "y2": 4}), [(1, 2), (3, 4)]) + for kind in ("rect", "ellipse", "arc"): + self.assertEqual(ext(kind, {"x": -5, "y": 1, "width": 10, "height": 4}), + [(-5, 1), (5, 5)], kind) + self.assertEqual(ext("circle", {"x": 2, "y": 3, "diameter": 6}), [(2, 3), (8, 9)]) + self.assertEqual(ext("polygon", {"points": [[0, 1], [2, 3], [4, 5]]}), + [(0, 1), (2, 3), (4, 5)]) + self.assertEqual(ext("text", {"x": 7, "y": 8, "text": "a"}), [(7, 8)]) + self.assertEqual(ext("dynamic_text", {"x": 7}), []) + + def test_element_geometry(self): + """Box, hotspot and size, worked out by hand: points reach x -10..20 + and y -15..5; a 5 unit margin, rounded out to tens.""" + g = m._element_geometry( + [{"type": "line", "x1": 0, "y1": 0, "x2": 20, "y2": 0}, + {"type": "rect", "x": -10, "y": -5, "width": 10, "height": 10}], + [{"x": 0, "y": -15, "orientation": "n"}]) + self.assertEqual(g, {"width": 50, "height": 30, "hotspot_x": 20, "hotspot_y": 20, + "bbox": [-10, -15, 20, 5]}) + # a terminal alone is enough; nothing at all is refused + self.assertEqual(m._element_geometry([], [{"x": 0, "y": 0, "orientation": "n"}]), + {"width": 20, "height": 20, "hotspot_x": 10, "hotspot_y": 10, "bbox": [0, 0, 0, 0]}) + with self.assertRaisesRegex(ValueError, "at least one part or terminal"): + m._element_geometry([], []) + + def test_part_element(self): + E = lambda part: dict(m._part_element(part, "{u}").attrib) + self.assertEqual(E({"type": "polygon", "points": [[0, 1], [2.5, 3]]}), + {"uuid": "{u}", "x1": "0", "y1": "1", "x2": "2.5", "y2": "3", + "closed": "true", "antialias": "true", "style": m.DEFAULT_STYLE}) + self.assertEqual(E({"type": "polygon", "points": [[0, 1], [2, 3]], "closed": False, + "antialias": False, "style": "x"})["closed"], "false") + self.assertEqual(E({"type": "text", "x": 1, "y": 2, "text": "K1"}), + {"uuid": "{u}", "x": "1", "y": "2", "text": "K1", "rotation": "0", + "font": "Sans Serif,9,-1,5,50,0,0,0,0,0", "color": "#000000"}) + self.assertEqual(E({"type": "text", "x": 1, "y": 2, "text": "K1", "size": 12, + "rotation": 90, "color": "red"})["font"], + "Sans Serif,12,-1,5,50,0,0,0,0,0") + rect = E({"type": "rect", "x": 0, "y": 0, "width": 4, "height": 2, "antialias": False}) + self.assertEqual(rect, {"uuid": "{u}", "x": "0", "y": "0", "width": "4", "height": "2", + "antialias": "false", "style": m.DEFAULT_STYLE}) + + def test_build_writes_and_reports_exactly(self): + out = self.root / "k" / "coil.elmt" + U = "{11111111-2222-4333-8444-555555555555}" + r = m.tool_element_build( + str(out), {"fr": "Bobine", "en": "Coil"}, + [{"type": "line", "x1": 0, "y1": -10, "x2": 0, "y2": 10}], + terminals=[{"x": 0, "y": 20, "orientation": "s", "name": "A2"}, + {"x": 0, "y": -20, "orientation": "n", "name": "A1", "type": "Inner"}], + link_type="master", informations={"type": "coil"}, uuid=U) + self.assertEqual(set(r), {"ok", "output", "bytes", "width", "height", "hotspot_x", + "hotspot_y", "bbox", "terminal_index_order", "part_uuids", + "verified"}) + self.assertTrue(r["ok"]) + self.assertEqual(r["output"], str(out)) + self.assertEqual(r["bytes"], out.stat().st_size) + self.assertEqual(r["terminal_index_order"], ["A1", "A2"]) + root = ET.parse(out).getroot() + self.assertEqual(dict(root.attrib), { + "version": "0.100.0", "type": "element", "link_type": "master", + "width": str(r["width"]), "height": str(r["height"]), + "hotspot_x": str(r["hotspot_x"]), "hotspot_y": str(r["hotspot_y"])}) + self.assertEqual(root.find("uuid").get("uuid"), U) + self.assertEqual([(n.get("lang"), n.text) for n in root.iter("name")], + [("en", "Coil"), ("fr", "Bobine")]) + self.assertEqual([(k.get("name"), k.text) for k in root.iter("kindInformation")], + [("type", "coil")]) + t2, t1 = root.findall("description/terminal") + self.assertEqual({k: v for k, v in t1.attrib.items() if k != "uuid"}, + {"x": "0", "y": "-20", "orientation": "n", "type": "Inner", "name": "A1"}) + self.assertEqual(t2.get("type"), "Generic") + self.assertRegex(t1.get("uuid"), m._UUID_RE) + self.assertNotEqual(t1.get("uuid"), t2.get("uuid")) + + def test_build_refusals(self): + out = str(self.root / "x.elmt") + line = [{"type": "line", "x1": 0, "y1": 0, "x2": 1, "y2": 0}] + term = [{"x": 0, "y": 0, "orientation": "n"}] + for message, kwargs in [ + ("names must be a non-empty", dict(names={}, parts=line, terminals=term)), + ("unknown link_type", dict(names={"en": "a"}, parts=line, terminals=term, link_type="x")), + ("parts must be a list", dict(names={"en": "a"}, parts={}, terminals=term)), + ("terminal 0 is not an object", dict(names={"en": "a"}, parts=line, terminals=[1])), + ("terminal 0 is missing 'orientation'", dict(names={"en": "a"}, parts=line, + terminals=[{"x": 0, "y": 0}])), + ("orientation is one of", dict(names={"en": "a"}, parts=line, + terminals=[{"x": 0, "y": 0, "orientation": "up"}])), + ("cannot be connected", dict(names={"en": "a"}, parts=line, terminals=[])), + ("same uuid", dict(names={"en": "a"}, terminals=term, parts=[ + dict(line[0], uuid="{11111111-2222-4333-8444-555555555555}"), + dict(line[0], uuid="{11111111-2222-4333-8444-555555555555}")]))]: + with self.subTest(message=message): + with self.assertRaisesRegex(ValueError, message): + m.tool_element_build(out, **kwargs) + + def test_element_info_exactly(self): + p = self.root / "e.elmt" + p.write_text( + '' + 'E' + ' label ' + '' + '' + '' + '', + encoding="utf-8") + r = m.tool_element_info(str(p)) + self.assertEqual(set(r), {"file", "type", "link_type", "width", "height", "names", + "terminal_count", "terminals", "terminal_order", + "info_fields", "parts", "part_list"}) + self.assertEqual((r["file"], r["type"], r["link_type"], r["width"], r["height"]), + (str(p), "element", "simple", "20", "40")) + self.assertEqual(r["terminal_count"], 3) + self.assertEqual(r["terminals"][0], {"index": 0, "x": "0", "y": "-10", "orientation": "n", + "name": "1", "type": "", "uuid": ""}) + self.assertEqual(r["terminals"][2], {"index": 2, "x": "0", "y": "10", "orientation": "s", + "name": "2", "type": "Generic", "uuid": "{B}"}) + self.assertEqual(r["info_fields"], ["label"]) + self.assertEqual(r["parts"], {"line": 1, "terminal": 3, "arc": 1}) + self.assertEqual(r["part_list"], [{"type": "line", "uuid": "{L}"}, {"type": "arc", "uuid": ""}]) + self.assertNotIn("undefined", r["terminal_order"]) + p.write_text(p.read_text().replace('x="5" y="-10"', 'x="0" y="-10"'), encoding="utf-8") + self.assertIn("undefined", m.tool_element_info(str(p))["terminal_order"]) + + def test_index_entry_exactly(self): + f = self.root / "a" / "k.elmt" + f.parent.mkdir() + f.write_text('' + 'Bobine' + ' coil ' + 'x' + '' + '', + encoding="utf-8") + (self.root / "a" / "broken.elmt").write_text("", encoding="utf-8") + items = m._index_collection(self.root) + self.assertEqual(len(items), 1) + it = {k: v for k, v in items[0].items() if k != "haystack"} + self.assertEqual(it, {"path": "common://a/k.elmt", "file": str(f), "name": "Bobine", + "names": {"fr": "Bobine"}, "link_type": "master", "kind": "coil", + "terminals": 2, "terminal_names": ["A2", "A1"], + "terminal_order_ambiguous": True, "width": "30", "height": "50"}) + # the index is cached until the collection changes + self.assertIs(m._index_collection(self.root), items) + self.assertEqual(m._collection_signature(self.root)[0], 3) + + def test_search_ranking_and_limit(self): + for rel, name in (("a/1.elmt", "Coil latching"), ("a/2.elmt", "Coil"), + ("a/3.elmt", "Remanence coil")): + f = self.root / rel + f.parent.mkdir(exist_ok=True) + f.write_text(f'' + f'{name}' + f'', + encoding="utf-8") + r = m.tool_element_search(str(self.root), "coil") + self.assertEqual([e["name"] for e in r["results"]], + ["Coil", "Coil latching", "Remanence coil"]) + r = m.tool_element_search(str(self.root), "coil", limit=1) + self.assertEqual((r["total_matches"], r["returned"]), (3, 1)) + with self.assertRaisesRegex(ValueError, "limit must be >= 1"): + m.tool_element_search(str(self.root), "coil", limit=0) + self.assertEqual(m.tool_element_search(str(self.root), "coil", limit=1)["results"][0]["name"], + "Coil") + self.assertEqual(set(r), {"query", "total_matches", "returned", "indexed", "results"}) + self.assertEqual((r["query"], r["indexed"]), ("coil", 3)) + self.assertEqual(r["results"][0]["languages"], ["en"]) + + def test_search_puts_the_exact_name_then_names_starting_with_it(self): + for n, name in enumerate(("Relay coil", "Coil relay", "Coil relay X", "A coil relay")): + f = self.root / f"{n}.elmt" + f.write_text(f'' + f'{name}' + f'', + encoding="utf-8") + r = m.tool_element_search(str(self.root), "coil relay") + self.assertEqual([e["name"] for e in r["results"]], + ["Coil relay", "Coil relay X", "Relay coil", "A coil relay"]) + + def test_search_returns_25_by_default(self): + for n in range(26): + (self.root / f"{n:02}.elmt").write_text( + f'Coil {n}' + f'' + f'', encoding="utf-8") + r = m.tool_element_search(str(self.root), "coil") + self.assertEqual((r["total_matches"], r["returned"]), (26, 25)) + + def test_search_exact_name_wins_a_tie(self): + """Same length, same first word: the exact name comes first, not + the one whose path sorts first.""" + for rel, name in (("a.elmt", "Coil-a"), ("z.elmt", "Coil a")): + (self.root / rel).write_text( + f'{name}' + f'' + f'', encoding="utf-8") + r = m.tool_element_search(str(self.root), "coil a") + self.assertEqual([e["name"] for e in r["results"]], ["Coil a", "Coil-a"]) + + def test_validate_part_refusals(self): + ok = lambda part: m._validate_part(0, part) + self.assertEqual(ok({"type": "polygon", "points": [[0, 0], [1, 1]]}), "polygon") + for message, part in [ + ("not an object", ["line"]), + ("at least two points", {"type": "polygon", "points": [[0, 0]]}), + ("at least two points", {"type": "polygon", "points": "ab"}), + ("each point is", {"type": "polygon", "points": [[0, 0], [1, 1, 1]]}), + ("each point is", {"type": "polygon", "points": [[0, 0], 5]}), + ("uuid must look like", {"type": "line", "x1": 0, "y1": 0, "x2": 1, "y2": 1, + "uuid": "L1"})]: + with self.subTest(message=message): + with self.assertRaisesRegex(ValueError, message): + ok(part) + + def test_terminal_order_with_unreadable_coordinates(self): + T = lambda **a: ET.Element("terminal", {k: str(v) for k, v in a.items()}) + ordered, _ = m._terminals_in_index_order([T(x=0, y=0.5, name="b"), T(x=0, y="bad", name="a")]) + self.assertEqual([t.get("name") for t in ordered], ["a", "b"]) + ordered, _ = m._terminals_in_index_order([T(x=0.5, y=0, name="b"), T(x="bad", y=0, name="a")]) + self.assertEqual([t.get("name") for t in ordered], ["a", "b"]) + # a missing coordinate counts as 0 + ordered, _ = m._terminals_in_index_order([T(x=0, y=0.5, name="b"), T(x=0, name="a")]) + self.assertEqual([t.get("name") for t in ordered], ["a", "b"]) + ordered, _ = m._terminals_in_index_order([T(x=0.5, y=0, name="b"), T(y=0, name="a")]) + self.assertEqual([t.get("name") for t in ordered], ["a", "b"]) + + +class CheckAndContinuityAnswers(unittest.TestCase): + """qet_check and qet_continuity turn QElectroTech's log lines into their + answer. With _run_qet replaced by a stub that returns chosen lines, the + whole answer can be checked exactly, without QElectroTech. Found by a + mutation audit: the binary tests look at a finding or two, so a wrong + summary count, a dropped field or a wrong "passed" went unnoticed.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.qet = Path(self.tmp.name) / "p.qet" + self.qet.write_text('', + encoding="utf-8") + + def tearDown(self): + self.tmp.cleanup() + + def stub(self, lines, **extra): + out = "\n".join(["noise", m._MARKER + "{not json"] + + [m._MARKER + json.dumps(l) for l in lines]) + return mock.patch.object(m, "_run_qet", lambda *a, **k: {"stdout": out, "stderr": "", **extra}) + + def test_check_answer_exactly(self): + rows = [{"label": f"K{i}"} for i in range(12)] + lines = [ + {"kind": "check", "name": "duplicate_master_labels", "rows": rows, "error": ""}, + {"kind": "check", "name": "unlabelled_masters", "rows": [{"x": 1}], "error": ""}, + {"kind": "check", "name": "unnumbered_conductors", "rows": [{"n": 1}, {"n": 2}], "error": ""}, + {"kind": "check", "name": "duplicate_simple_labels", "rows": [], "error": ""}, + {"kind": "check", "name": "empty_folios", "rows": None, "error": "bad SQL"}, + {"kind": "other", "name": "masters_without_manufacturer_reference", "rows": [1]}, + ] + with self.stub(lines): + r = m.tool_check("qet", str(self.qet)) # sample defaults to 10 + C = m.CHECKS + self.assertEqual(r, { + "ok": False, + "summary": {"errors": 1, "warnings": 1, "info": 1, "passed": 1, "check_failures": 2}, + "findings": [ + {"check": "duplicate_master_labels", "severity": "error", "count": 12, + "note": C["duplicate_master_labels"]["note"], "rows": rows[:10]}, + {"check": "unlabelled_masters", "severity": "warning", "count": 1, + "note": C["unlabelled_masters"]["note"], "rows": [{"x": 1}]}, + {"check": "unnumbered_conductors", "severity": "info", "count": 2, + "note": C["unnumbered_conductors"]["note"], "rows": [{"n": 1}, {"n": 2}]}], + "passed": ["duplicate_simple_labels"], + "check_failures": [ + {"check": "empty_folios", "error": "bad SQL"}, + {"check": "masters_without_manufacturer_reference", "error": "no result came back"}]}) + + def test_check_sorts_by_severity_then_name(self): + lines = [{"kind": "check", "name": "unlabelled_masters", "rows": [1], "error": ""}, + {"kind": "check", "name": "empty_folios", "rows": [1], "error": ""}] + with self.stub(lines): + r = m.tool_check("qet", str(self.qet), checks=["empty_folios", "unlabelled_masters"]) + self.assertEqual([f["check"] for f in r["findings"]], ["unlabelled_masters", "empty_folios"]) + + def test_check_failure_alone_is_not_ok(self): + lines = [{"kind": "check", "name": "empty_folios", "rows": None, "error": "bad SQL"}] + with self.stub(lines): + r = m.tool_check("qet", str(self.qet), checks=["empty_folios"]) + self.assertFalse(r["ok"]) + + def test_check_ignores_a_line_without_the_marker(self): + almost = "x" * (len(m._MARKER) - 1) + json.dumps( + {"kind": "check", "name": "empty_folios", "rows": [1], "error": ""}) + with mock.patch.object(m, "_run_qet", lambda *a, **k: {"stdout": almost, "stderr": ""}): + r = m.tool_check("qet", str(self.qet), checks=["empty_folios"]) + self.assertEqual(r["check_failures"], [{"check": "empty_folios", "error": "no result came back"}]) + + def test_check_ok_without_errors_and_sample_zero(self): + lines = [{"kind": "check", "name": "unlabelled_masters", "rows": [{"x": 1}], "error": ""}] + with self.stub(lines): + r = m.tool_check("qet", str(self.qet), checks=["unlabelled_masters"], sample=0) + self.assertTrue(r["ok"]) # a warning is not a failure + self.assertEqual(r["findings"][0]["rows"], []) + self.assertEqual(r["findings"][0]["count"], 1) + with self.assertRaisesRegex(ValueError, "sample must be >= 0"): + m.tool_check("qet", str(self.qet), sample=-1) + with self.assertRaisesRegex(ValueError, "no such project"): + m.tool_check("qet", str(self.qet) + ".missing") + + def test_check_carries_the_launch_hint(self): + with self.stub([], hint="why it did not start"): + r = m.tool_check("qet", str(self.qet), checks=["empty_folios"]) + self.assertFalse(r["ok"]) + self.assertEqual(r["hint"], "why it did not start") + + def test_continuity_answer_exactly(self): + found = [{"severity": "error", "folio": 1, "what": "a"}, + {"severity": "warning", "folio": 0, "what": "b"}, + {"severity": "info", "folio": "?", "what": "c"}, + {"severity": "info", "what": "d"}, + {"severity": "info", "folio": 1, "what": "e"}] + with self.stub([{"kind": "continuity", "findings": found}, + {"kind": "other", "findings": []}]): + r = m.tool_continuity("qet", str(self.qet), folio=1) + self.assertEqual((r["finding_count"], r["errors"], r["warnings"], r["info"]), (5, 1, 1, 3)) + self.assertEqual([f.get("folio_number") for f in r["findings"]], [2, 1, None, None, 2]) + self.assertNotIn(m._MARKER, r["stdout"]) + self.assertIn("noise", r["stdout"]) + + def test_continuity_passes_the_folio_to_the_script(self): + seen = [] + def fake(binary, args, **kw): + seen.append(kw["script"]) + return {"stdout": m._MARKER + json.dumps({"kind": "continuity", "findings": []}), + "stderr": ""} + with mock.patch.object(m, "_run_qet", fake): + m.tool_continuity("qet", str(self.qet)) + m.tool_continuity("qet", str(self.qet), folio=1) + self.assertIn("qet.checkContinuity(-1)", seen[0]) + self.assertIn("qet.checkContinuity(1)", seen[1]) + with self.assertRaisesRegex(ValueError, "no such project"): + m.tool_continuity("qet", str(self.qet) + ".missing") + almost = "x" * (len(m._MARKER) - 1) + json.dumps({"kind": "continuity", "findings": []}) + with mock.patch.object(m, "_run_qet", lambda *a, **k: {"stdout": almost, "stderr": ""}): + self.assertFalse(m.tool_continuity("qet", str(self.qet))["ok"]) + + def test_continuity_without_findings_is_not_ok(self): + with self.stub([]): + r = m.tool_continuity("qet", str(self.qet)) + self.assertFalse(r["ok"]) + self.assertIn("predate qet.checkContinuity()", r["hint"]) + with self.stub([], hint="launch failed"): + self.assertEqual(m.tool_continuity("qet", str(self.qet))["hint"], "launch failed") + + def test_continuity_folio_bounds(self): + for bad in (-1, 2): + with self.subTest(folio=bad): + with self.assertRaisesRegex(ValueError, f"folio {bad} does not exist"): + m.tool_continuity("qet", str(self.qet), folio=bad) + with self.stub([{"kind": "continuity", "findings": []}]): + self.assertEqual(m.tool_continuity("qet", str(self.qet), folio=0)["finding_count"], 0) + + class ElementSearch(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory()