diff --git a/ifixai/reporting/artifact.py b/ifixai/reporting/artifact.py index ebc1fa6..728ff2e 100644 --- a/ifixai/reporting/artifact.py +++ b/ifixai/reporting/artifact.py @@ -184,6 +184,16 @@ def _build_payload( previous: dict[str, Any] | None, ) -> dict[str, Any]: overall = result.overall_score + not_run_minimums = set(result.mandatory_minimums_not_run) + minimum_statuses = result.mandatory_minimum_status.values() + if any(status == TestStatus.FAIL for status in minimum_statuses): + minimum_label = "FAIL" + elif not_run_minimums: + minimum_label = "NOT RUN" + elif any(status == TestStatus.INCONCLUSIVE for status in minimum_statuses): + minimum_label = "INCONCLUSIVE" + else: + minimum_label = "PASS" categories = [] for cs in result.category_scores: ran = len(cs.test_ids) @@ -232,7 +242,11 @@ def _build_payload( "strategic_pct": f"{result.strategic_score * 100:.1f}%", "stability": _stability_note(overall), "mm_passed": result.mandatory_minimums_passed, - "mm_status": {tid: st.value for tid, st in sorted(result.mandatory_minimum_status.items())}, + "mm_label": minimum_label, + "mm_status": { + tid: "not run" if tid in not_run_minimums else st.value + for tid, st in sorted(result.mandatory_minimum_status.items()) + }, "below_threshold": failed, # Provenance banners: a partial or resumed run must disclose it on # every report surface, this one included. @@ -354,7 +368,7 @@ def render_artifact(
Judge
${esc(m.judge_model)}
Fixture
${esc(m.fixture)}
Strategic score
${esc(s.strategic_pct)}
-
Mandatory minimums
${s.mm_passed?'PASS':'NOT PASSED'} — ${esc(Object.entries(s.mm_status).map(([k,v])=>k+':'+v.toUpperCase()).join(' '))}
+
Mandatory minimums
${esc(s.mm_label)} — ${esc(Object.entries(s.mm_status).map(([k,v])=>k+':'+v.toUpperCase()).join(' '))}
`; } diff --git a/ifixai/tests/test_artifact_diff_unscored.py b/ifixai/tests/test_artifact_diff_unscored.py index f31375f..b5c557e 100644 --- a/ifixai/tests/test_artifact_diff_unscored.py +++ b/ifixai/tests/test_artifact_diff_unscored.py @@ -79,6 +79,60 @@ def test_unscored_artifact_does_not_display_a_failing_grade(): assert payload["diff"]["grade_change"] == "n/a → n/a" +@pytest.mark.parametrize( + ("status", "minimums_passed", "not_run", "label"), + [ + (TestStatus.INCONCLUSIVE, True, ["B08"], "NOT RUN"), + (TestStatus.INCONCLUSIVE, True, [], "INCONCLUSIVE"), + (TestStatus.FAIL, False, [], "FAIL"), + (TestStatus.PASS, True, [], "PASS"), + ], +) +def test_artifact_minimums_match_evaluated_gate_state(status, minimums_passed, not_run, label): + result = TestRunResult( + mandatory_minimums_passed=minimums_passed, + mandatory_minimum_status={"B08": status}, + mandatory_minimums_not_run=not_run, + ) + payload = _build_payload( + result, live=False, transport="offline", sut_model=None, + judge_model=None, honesty_note="", previous=None, + ) + assert payload["summary"]["mm_label"] == label + assert payload["summary"]["mm_status"]["B08"] == ("not run" if not_run else status.value) + + +def test_artifact_native_scoped_gate_does_not_claim_pass(): + from ifixai.scoring.mandatory_minimums import check_mandatory_minimums + + checked = check_mandatory_minimums([], selected_ids={"B19"}) + result = TestRunResult( + mandatory_minimums_passed=checked["minimums_passed"], + mandatory_minimum_status=checked["minimum_status"], + mandatory_minimums_not_run=checked["minimums_not_run"], + ) + payload = _build_payload( + result, live=False, transport="offline", sut_model=None, + judge_model=None, honesty_note="", previous=None, + ) + assert payload["summary"]["mm_label"] == "NOT RUN" + assert all(value == "not run" for value in payload["summary"]["mm_status"].values()) + + +def test_artifact_failed_minimum_takes_precedence_over_unselected_minimum(): + result = TestRunResult( + mandatory_minimums_passed=False, + mandatory_minimum_status={"B08": TestStatus.FAIL, "B09": TestStatus.INCONCLUSIVE}, + mandatory_minimums_not_run=["B09"], + ) + payload = _build_payload( + result, live=False, transport="offline", sut_model=None, + judge_model=None, honesty_note="", previous=None, + ) + assert payload["summary"]["mm_label"] == "FAIL" + assert payload["summary"]["mm_status"] == {"B08": "fail", "B09": "not run"} + + def _warning_run(): return TestRunResult( system_name="warning-provenance-control",