From 6ecbbd854ea69fcf3de35237ab430515393c78b1 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Wed, 25 Feb 2026 01:44:39 +0530 Subject: [PATCH] test: add phase-2 cross-runtime conformance mapping baseline --- tests/protocol_fixtures/PROTOCOL_FIXTURES.md | 7 + .../cross_runtime_matrix.phase2.json | 211 ++++++++++++++++++ tests/test_protocol_conformance_phase2.py | 57 +++++ 3 files changed, 275 insertions(+) create mode 100644 tests/protocol_fixtures/cross_runtime_matrix.phase2.json create mode 100644 tests/test_protocol_conformance_phase2.py diff --git a/tests/protocol_fixtures/PROTOCOL_FIXTURES.md b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md index 0b85c8b..1bf953e 100644 --- a/tests/protocol_fixtures/PROTOCOL_FIXTURES.md +++ b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md @@ -4,9 +4,16 @@ This directory contains the phase-1 protocol conformance baseline for Python. - `schema.phase1.json`: fixture document shape and supported case targets. - `python_phase1_cases.json`: initial baseline cases (report-only mode metadata). +- `cross_runtime_matrix.phase2.json`: phase-2 cross-runtime mapping matrix in report-only mode. Phase-1 scope: - No runtime behavior changes. - Python-only execution through `tests/test_protocol_conformance.py`. - Fixture format is language-neutral to enable future cross-binding runners. + +Phase-2 scope (mapping only): + +- No runtime behavior changes. +- Adds a cross-runtime matrix to track per-case audit status and classification. +- Keeps CI non-blocking for non-Python runtimes by marking them as `not_audited` until adapters are added. diff --git a/tests/protocol_fixtures/cross_runtime_matrix.phase2.json b/tests/protocol_fixtures/cross_runtime_matrix.phase2.json new file mode 100644 index 0000000..746f2f1 --- /dev/null +++ b/tests/protocol_fixtures/cross_runtime_matrix.phase2.json @@ -0,0 +1,211 @@ +{ + "schema_version": "1.0", + "phase": "2", + "mode": "report_only", + "source_fixture": "python_phase1_cases.json", + "runtimes": [ + "python", + "cpp", + "matlab", + "octave", + "verilog" + ], + "classifications": [ + "required", + "implementation_defined", + "known_deviation" + ], + "statuses": [ + "observed_pass", + "observed_fail", + "not_audited" + ], + "cases": [ + { + "id": "parse_params/simple_types_and_whitespace", + "target": "parse_params", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "parse_params/embedded_equals_not_split", + "target": "parse_params", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "initval/valid_list_sets_simtime", + "target": "initval", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "initval/invalid_input_returns_empty_and_preserves_simtime", + "target": "initval", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "write_zmq/list_payload_prepends_timestamp_without_mutation", + "target": "write_zmq", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "write_zmq/non_list_payload_forwarded_as_is", + "target": "write_zmq", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + } + ] +} diff --git a/tests/test_protocol_conformance_phase2.py b/tests/test_protocol_conformance_phase2.py new file mode 100644 index 0000000..01d8c3f --- /dev/null +++ b/tests/test_protocol_conformance_phase2.py @@ -0,0 +1,57 @@ +import json +from pathlib import Path + + +FIXTURE_DIR = Path(__file__).parent / "protocol_fixtures" +PHASE1_CASES_PATH = FIXTURE_DIR / "python_phase1_cases.json" +PHASE2_MATRIX_PATH = FIXTURE_DIR / "cross_runtime_matrix.phase2.json" + +EXPECTED_RUNTIMES = {"python", "cpp", "matlab", "octave", "verilog"} +EXPECTED_CLASSIFICATIONS = {"required", "implementation_defined", "known_deviation"} +EXPECTED_STATUSES = {"observed_pass", "observed_fail", "not_audited"} + + +def _load_json(path): + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def _phase1_cases(): + doc = _load_json(PHASE1_CASES_PATH) + return {case["id"]: case for case in doc["cases"]} + + +def _phase2_matrix(): + return _load_json(PHASE2_MATRIX_PATH) + + +def test_phase2_matrix_metadata_and_enums(): + doc = _phase2_matrix() + assert doc["phase"] == "2" + assert doc["mode"] == "report_only" + assert doc["source_fixture"] == "python_phase1_cases.json" + assert set(doc["runtimes"]) == EXPECTED_RUNTIMES + assert set(doc["classifications"]) == EXPECTED_CLASSIFICATIONS + assert set(doc["statuses"]) == EXPECTED_STATUSES + + +def test_phase2_matrix_covers_all_phase1_cases(): + phase1 = _phase1_cases() + matrix_cases = _phase2_matrix()["cases"] + matrix_ids = {case["id"] for case in matrix_cases} + assert matrix_ids == set(phase1.keys()) + + +def test_phase2_matrix_rows_have_consistent_shape(): + phase1 = _phase1_cases() + for row in _phase2_matrix()["cases"]: + assert row["id"] in phase1 + assert row["target"] == phase1[row["id"]]["target"] + assert set(row["runtime_results"].keys()) == EXPECTED_RUNTIMES + + for runtime, result in row["runtime_results"].items(): + assert result["status"] in EXPECTED_STATUSES + assert result["classification"] in EXPECTED_CLASSIFICATIONS + assert isinstance(result["note"], str) and result["note"].strip() + if runtime == "python": + assert result["status"] == "observed_pass"