fix(reporting): preserve grading provenance in human exports (#201)

Signed-off-by: Rudy Celekli <47457359+rudycelekli@users.noreply.github.com>
Co-authored-by: n-papaioannou <n.papaioannou@ime.life>

Author: rudycelekli

ifixai/reporting/artifact.py

@@ -78,6 +78,8 @@ def _evidence_payload(ev) -> dict[str, Any]:
         "expected": _scrub(ev.expected or ev.expected_behavior or ""),
         "actual": _scrub(ev.actual_response or ev.actual or ""),
         "evaluation_result": ev.evaluation_result or "",
+        "extraction_error": ev.extraction_error.value if ev.extraction_error else None,
+        "is_diagnostic": ev.is_diagnostic,
         "passed": ev.passed,
         "evaluation_method": ev.evaluation_method.value,
         "verdict": verdict_label,
@@ -242,6 +244,8 @@ def _build_payload(
         },
         "categories": categories,
         "compliance": compliance,
+        "warnings": [_scrub(warning) for warning in result.warnings],
+        "validation_warnings": [_scrub(warning) for warning in result.validation_warnings],
         "checks": [_check_payload(br) for br in sorted(result.test_results, key=lambda b: b.test_id)],
         "diff": _diff_payload(result, previous) if previous else None,
     }
@@ -354,6 +358,12 @@ def render_artifact(
   </div>`;
 }
 
+function warningBanners(){
+  const render = (warnings, kind) => (warnings||[])
+    .map(w => `<div class="banner ${kind}">${esc(w)}</div>`).join('');
+  return render(D.validation_warnings, 'bad') + render(D.warnings, 'warn');
+}
+
 function categories(){
   if(!D.categories.length) return '';
   const rows = D.categories.map(c=>`<tr><td>${esc(c.category)}</td><td>${esc(c.score_pct)}</td><td>${esc((c.weight*100).toFixed(0))}%</td><td>${esc(c.coverage)}</td></tr>`).join('');
@@ -379,12 +389,15 @@ def render_artifact(
 }
 
 function evidence(ev){
+  const label = ev.is_diagnostic ? 'diagnostic' : ev.extraction_error ? 'grading unavailable' : ev.passed ? 'pass' : 'fail';
+  const tagClass = ev.is_diagnostic || ev.extraction_error ? 'inconclusive' : label;
   const conf = ev.confidence==null?'':` · confidence ${(ev.confidence*100).toFixed(0)}%`;
   const dims = (ev.dimensions||[]).map(dm=>`<tr><td>${dm.passed?'✓':'✗'} ${esc(dm.name)}${dm.mandatory?' <span class="dim">(mandatory)</span>':''}</td><td>${dm.confidence==null?'':(dm.confidence*100).toFixed(0)+'%'}</td><td class="dim">${esc(dm.reasoning)}</td></tr>`).join('');
   const fld=(label,val)=> val? `<div class="kvp"><b>${label}</b><pre>${esc(val)}</pre></div>`:'';
   return `<div class="ev">
-    <div class="evhead"><span class="tag ${ev.passed?'pass':'fail'}">${ev.passed?'pass':'fail'}</span><span>${esc(ev.evaluation_method)}${conf}</span></div>
+    <div class="evhead"><span class="tag ${tagClass}">${label}</span><span>${esc(ev.evaluation_method)}${conf}</span></div>
     ${ev.description?`<div class="kvp dim">${esc(ev.description)}</div>`:''}
+    ${ev.extraction_error?fld('Grading unavailable',ev.extraction_error):''}
     ${fld('Prompt used',ev.prompt)}
     ${fld('Expected',ev.expected)}
     ${fld('Actual',ev.actual)}
@@ -455,6 +468,7 @@ def render_artifact(
 
 const app = document.getElementById('app');
 app.insertAdjacentHTML('beforeend', header());
+app.insertAdjacentHTML('beforeend', warningBanners());
 const diff = diffSection(); if(diff) app.insertAdjacentHTML('beforeend', diff);
 app.insertAdjacentHTML('beforeend', categories());
 app.insertAdjacentHTML('beforeend', compliance());

ifixai/reporting/scorecard.py

@@ -425,6 +425,13 @@ def render_not_run_section(result: TestRunResult) -> str:
     return "\n".join(lines)
 
 
+def render_run_warnings(result: TestRunResult) -> str:
+    """Keep operator and grading caveats with exported results."""
+    if not result.warnings:
+        return ""
+    return "## Run Warnings\n\n" + "\n".join(f"- {warning}" for warning in result.warnings)
+
+
 def generate_markdown_report(result: TestRunResult) -> str:
     frameworks = load_all_mappings()
 
@@ -436,6 +443,7 @@ def generate_markdown_report(result: TestRunResult) -> str:
         render_insights(result),
         render_category_table(result),
         render_mandatory_minimums(result),
+        render_run_warnings(result),
         render_consistency_warnings(result),
         render_test_table(result),
         render_not_run_section(result),

ifixai/tests/test_artifact_diff_unscored.py

@@ -77,3 +77,69 @@ def test_unscored_artifact_does_not_display_a_failing_grade():
     assert payload["summary"]["grade"] == "n/a"
     assert payload["summary"]["grade_class"] == "inconclusive"
     assert payload["diff"]["grade_change"] == "n/a → n/a"
+
+
+def _warning_run():
+    return TestRunResult(
+        system_name="warning-provenance-control",
+        warnings=["Pinned seeds: memorization resistance reduced", "Judge substituted after error"],
+        validation_warnings=["run_invalid: ignore the grade", "judge_health: verdict contract failures"],
+    )
+
+
+def test_markdown_preserves_operator_warnings():
+    from ifixai.reporting.scorecard import generate_markdown_report
+
+    result = _warning_run()
+    rendered = generate_markdown_report(result)
+    for warning in [*result.warnings, *result.validation_warnings]:
+        assert warning in rendered
+
+
+def test_artifact_preserves_operator_and_measurement_warnings():
+    result = _warning_run()
+    payload = _build_payload(
+        result, live=True, transport="http", sut_model=None,
+        judge_model=None, honesty_note="", previous=None,
+    )
+    assert payload["warnings"] == result.warnings
+    assert payload["validation_warnings"] == result.validation_warnings
+
+
+def test_artifact_warning_data_cannot_terminate_embedded_script():
+    from ifixai.reporting.artifact import render_artifact
+
+    warning = "Controlled warning </script><img src=x onerror=alert(1)>"
+    rendered = render_artifact(
+        TestRunResult(warnings=[warning]), live=True, transport="http",
+        sut_model=None, judge_model=None, honesty_note="",
+    )
+    assert warning not in rendered
+    assert "Controlled warning \\u003c/script>" in rendered
+
+
+@pytest.mark.parametrize("error_kind", ["communication", "contract", "extraction", "budget"])
+def test_artifact_evidence_retains_unscorable_cause(error_kind):
+    from ifixai.core.types import EvidenceItem, JudgeErrorKind
+    from ifixai.reporting.artifact import _evidence_payload
+
+    item = EvidenceItem(
+        test_case_id="owned-ungraded-probe", passed=False,
+        actual_response="A reply without a usable grading result",
+        extraction_error=JudgeErrorKind(error_kind),
+    )
+    payload = _evidence_payload(item)
+    assert payload["extraction_error"] == error_kind
+    assert payload["is_diagnostic"] is False
+
+
+def test_artifact_evidence_retains_diagnostic_marker():
+    from ifixai.core.types import EvidenceItem
+    from ifixai.reporting.artifact import _evidence_payload
+
+    payload = _evidence_payload(EvidenceItem(
+        test_case_id="owned-run-diagnostic", passed=False, is_diagnostic=True,
+        description="Transport diagnostics, excluded from scoring",
+    ))
+    assert payload["is_diagnostic"] is True
+    assert payload["extraction_error"] is None