fix: reject partial and unscored results in CI example (#136)

Co-authored-by: stefyi-4355 <as.peter@ime.life>

Author: rudycelekli

ifixai/examples/ci_check.py

@@ -47,8 +47,10 @@ async def run_ci_check() -> int:
             f.write(generate_json_report(result))
         print(f"Report written to {config['report_path']}")
 
-    print(f"Grade:          {result.grade.value}")
-    print(f"Overall Score:  {result.overall_score:.0%}")
+    grade = result.grade.value if result.overall_score is not None else "n/a"
+    overall = f"{result.overall_score:.0%}" if result.overall_score is not None else "n/a"
+    print(f"Grade:          {grade}")
+    print(f"Overall Score:  {overall}")
     print(f"Strategic Score: {result.strategic_score:.0%}")
 
     passed_count = sum(1 for br in result.test_results if br.passed)
@@ -60,7 +62,15 @@ async def run_ci_check() -> int:
 
     is_passing = True
 
-    if result.overall_score < min_score:
+    if result.partial:
+        reason = f": {result.abort_reason}" if result.abort_reason else ""
+        print(f"\nFAIL: Partial run{reason}; finish the run before evaluating CI thresholds")
+        is_passing = False
+
+    if result.overall_score is None:
+        print("\nFAIL: Overall score unavailable; the run is not gradeable")
+        is_passing = False
+    elif result.overall_score < min_score:
         print(f"\nFAIL: Overall score {result.overall_score:.0%} < {min_score:.0%}")
         is_passing = False
 

ifixai/tests/test_ci_check_unscored.py

@@ -0,0 +1,62 @@
+import asyncio
+
+import pytest
+
+from ifixai.core.types import TestGrade, TestRunResult
+from ifixai.examples import ci_check
+
+
+def _config() -> dict[str, str]:
+    return {
+        "provider": "mock",
+        "api_key": "test-key",
+        "fixture": "test-fixture",
+        "model": "",
+        "system_name": "test-system",
+        "system_version": "1.0",
+        "min_score": "0.70",
+        "min_strategic": "0.80",
+        "report_path": "",
+    }
+
+
+@pytest.mark.parametrize(
+    ("overall_score", "partial", "expected_message"),
+    [
+        (None, False, "Overall Score:  n/a"),
+        (0.95, True, "partial run"),
+    ],
+)
+def test_ci_check_fails_cleanly_when_run_is_not_gradeable(
+    monkeypatch, capsys, overall_score, partial, expected_message
+):
+    async def fake_run_inspections(**kwargs):
+        return TestRunResult(
+            overall_score=overall_score,
+            strategic_score=0.95,
+            grade=TestGrade.A,
+            mandatory_minimums_passed=True,
+            partial=partial,
+            abort_reason="judge quota exhausted" if partial else None,
+        )
+
+    monkeypatch.setattr(ci_check, "read_env_config", _config)
+    monkeypatch.setattr(ci_check, "run_inspections", fake_run_inspections)
+
+    assert asyncio.run(ci_check.run_ci_check()) == 1
+    assert expected_message.lower() in capsys.readouterr().out.lower()
+
+
+def test_ci_check_still_passes_a_complete_scored_run(monkeypatch):
+    async def fake_run_inspections(**kwargs):
+        return TestRunResult(
+            overall_score=0.95,
+            strategic_score=0.95,
+            grade=TestGrade.A,
+            mandatory_minimums_passed=True,
+        )
+
+    monkeypatch.setattr(ci_check, "read_env_config", _config)
+    monkeypatch.setattr(ci_check, "run_inspections", fake_run_inspections)
+
+    assert asyncio.run(ci_check.run_ci_check()) == 0