Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions ifixai/reporting/artifact.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,16 @@ def _build_payload(
previous: dict[str, Any] | None,
) -> dict[str, Any]:
overall = result.overall_score
not_run_minimums = set(result.mandatory_minimums_not_run)
minimum_statuses = result.mandatory_minimum_status.values()
if any(status == TestStatus.FAIL for status in minimum_statuses):
minimum_label = "FAIL"
elif not_run_minimums:
minimum_label = "NOT RUN"
elif any(status == TestStatus.INCONCLUSIVE for status in minimum_statuses):
minimum_label = "INCONCLUSIVE"
else:
minimum_label = "PASS"
categories = []
for cs in result.category_scores:
ran = len(cs.test_ids)
Expand Down Expand Up @@ -232,7 +242,11 @@ def _build_payload(
"strategic_pct": f"{result.strategic_score * 100:.1f}%",
"stability": _stability_note(overall),
"mm_passed": result.mandatory_minimums_passed,
"mm_status": {tid: st.value for tid, st in sorted(result.mandatory_minimum_status.items())},
"mm_label": minimum_label,
"mm_status": {
tid: "not run" if tid in not_run_minimums else st.value
for tid, st in sorted(result.mandatory_minimum_status.items())
},
"below_threshold": failed,
# Provenance banners: a partial or resumed run must disclose it on
# every report surface, this one included.
Expand Down Expand Up @@ -354,7 +368,7 @@ def render_artifact(
<div>Judge</div><div>${esc(m.judge_model)}</div>
<div>Fixture</div><div>${esc(m.fixture)}</div>
<div>Strategic score</div><div>${esc(s.strategic_pct)}</div>
<div>Mandatory minimums</div><div>${s.mm_passed?'PASS':'NOT PASSED'} — ${esc(Object.entries(s.mm_status).map(([k,v])=>k+':'+v.toUpperCase()).join(' '))}</div>
<div>Mandatory minimums</div><div>${esc(s.mm_label)} — ${esc(Object.entries(s.mm_status).map(([k,v])=>k+':'+v.toUpperCase()).join(' '))}</div>
</div>`;
}

Expand Down
54 changes: 54 additions & 0 deletions ifixai/tests/test_artifact_diff_unscored.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,60 @@ def test_unscored_artifact_does_not_display_a_failing_grade():
assert payload["diff"]["grade_change"] == "n/a → n/a"


@pytest.mark.parametrize(
("status", "minimums_passed", "not_run", "label"),
[
(TestStatus.INCONCLUSIVE, True, ["B08"], "NOT RUN"),
(TestStatus.INCONCLUSIVE, True, [], "INCONCLUSIVE"),
(TestStatus.FAIL, False, [], "FAIL"),
(TestStatus.PASS, True, [], "PASS"),
],
)
def test_artifact_minimums_match_evaluated_gate_state(status, minimums_passed, not_run, label):
result = TestRunResult(
mandatory_minimums_passed=minimums_passed,
mandatory_minimum_status={"B08": status},
mandatory_minimums_not_run=not_run,
)
payload = _build_payload(
result, live=False, transport="offline", sut_model=None,
judge_model=None, honesty_note="", previous=None,
)
assert payload["summary"]["mm_label"] == label
assert payload["summary"]["mm_status"]["B08"] == ("not run" if not_run else status.value)


def test_artifact_native_scoped_gate_does_not_claim_pass():
from ifixai.scoring.mandatory_minimums import check_mandatory_minimums

checked = check_mandatory_minimums([], selected_ids={"B19"})
result = TestRunResult(
mandatory_minimums_passed=checked["minimums_passed"],
mandatory_minimum_status=checked["minimum_status"],
mandatory_minimums_not_run=checked["minimums_not_run"],
)
payload = _build_payload(
result, live=False, transport="offline", sut_model=None,
judge_model=None, honesty_note="", previous=None,
)
assert payload["summary"]["mm_label"] == "NOT RUN"
assert all(value == "not run" for value in payload["summary"]["mm_status"].values())


def test_artifact_failed_minimum_takes_precedence_over_unselected_minimum():
result = TestRunResult(
mandatory_minimums_passed=False,
mandatory_minimum_status={"B08": TestStatus.FAIL, "B09": TestStatus.INCONCLUSIVE},
mandatory_minimums_not_run=["B09"],
)
payload = _build_payload(
result, live=False, transport="offline", sut_model=None,
judge_model=None, honesty_note="", previous=None,
)
assert payload["summary"]["mm_label"] == "FAIL"
assert payload["summary"]["mm_status"] == {"B08": "fail", "B09": "not run"}


def _warning_run():
return TestRunResult(
system_name="warning-provenance-control",
Expand Down
Loading