Merge branch 'main' into docs/readme-rebrand

Author: n-papaioannou

docs/testing-your-agent.md

@@ -73,6 +73,11 @@ Atomic-claim verdicts accept JSON `true`/`false` and the strings `"true"`/`"fals
 output. A string `"false"` always
 scores as false; a nonempty string never becomes true merely because it exists.
 
+Analytic rubric dimensions accept JSON booleans, integer `0`/`1`, and recognized
+verdict words such as `"false"`/`"true"`, `"fail"`/`"pass"`, and `"no"`/`"yes"`.
+Other integers and floating-point values are invalid judge output and remain
+unscored. This normalization applies to every supported rubric envelope.
+
 `--eval-mode self` is a smoke test, flagged `self-judge bias`. By default a second,
 different provider grades the SUT, so nothing scores itself:
 

ifixai/cli/reports.py

@@ -1,5 +1,9 @@
+import os
 import re
+import stat
+from contextlib import suppress
 from pathlib import Path
+from uuid import uuid4
 
 import click
 
@@ -19,9 +23,33 @@ def _slugify(value: str) -> str:
     return cleaned or "unknown"
 
 
+def _write_report_atomic(path: Path, content: str) -> None:
+    mode = stat.S_IMODE(path.stat().st_mode) if path.exists() else None
+    temporary = path.parent / f".ifixai-report-{uuid4().hex}.tmp"
+    # New exports follow normal file-creation permissions, including the umask.
+    # Replacements stay private until the existing target mode is restored.
+    descriptor = os.open(
+        temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL,
+        0o666 if mode is None else 0o600,
+    )
+    os.close(descriptor)
+    try:
+        temporary.write_text(content, encoding="utf-8")
+        if mode is not None:
+            temporary.chmod(mode)
+        os.replace(temporary, path)
+    finally:
+        with suppress(OSError):
+            temporary.unlink(missing_ok=True)
+
+
 def save_reports(
-    result: TestRunResult, output_dir: str, report_format: str, run_nonce: str | None = None,
-    *, run_id: str | None = None,
+    result: TestRunResult,
+    output_dir: str,
+    report_format: str,
+    run_nonce: str | None = None,
+    *,
+    run_id: str | None = None,
 ) -> None:
     out_path = Path(output_dir)
     out_path.mkdir(parents=True, exist_ok=True)
@@ -39,14 +67,14 @@ def save_reports(
 
     if report_format in ("markdown", "both"):
         summary_path = out_path / f"{base_name}-summary.md"
-        summary_path.write_text(generate_summary_report(result), encoding="utf-8")
+        _write_report_atomic(summary_path, generate_summary_report(result))
         click.echo(f"  Summary (start here): {summary_path}")
 
         md_path = out_path / f"{base_name}.md"
-        md_path.write_text(generate_markdown_report(result), encoding="utf-8")
+        _write_report_atomic(md_path, generate_markdown_report(result))
         click.echo(f"  Full report:          {md_path}")
 
     if report_format in ("json", "both"):
         json_path = out_path / f"{base_name}.json"
-        json_path.write_text(generate_json_report(result), encoding="utf-8")
+        _write_report_atomic(json_path, generate_json_report(result))
         click.echo(f"  JSON (machine):       {json_path}")

ifixai/cli/run.py

@@ -210,13 +210,14 @@ def _resolve_concurrency(flag_value: int | None, no_parallel: bool) -> int:
 
 
 def _cfg_value(ctx: click.Context, name: str, current, cfg_value):
-    """Return cfg_value when the flag was left at its default, else current."""
+    """Use a saved default through its CLI type, preserving explicit flags."""
     from click.core import ParameterSource
 
     if cfg_value is None:
         return current
     if ctx.get_parameter_source(name) == ParameterSource.DEFAULT:
-        return cfg_value
+        parameter = next(param for param in ctx.command.params if param.name == name)
+        return parameter.type_cast_value(ctx, cfg_value)
     return current
 
 
@@ -332,6 +333,12 @@ async def _probe_then_close(
         await _aclose_provider(provider)
 
 
+def _validate_min_score(ctx: click.Context, param: click.Parameter, value: float) -> float:
+    if not 0 <= value <= 1:
+        raise click.BadParameter("must be a finite number between 0 and 1", ctx=ctx, param=param)
+    return value
+
+
 @click.command()
 @click.option(
     "--provider",
@@ -494,6 +501,7 @@ async def _probe_then_close(
 @click.option(
     "--min-score",
     type=float,
+    callback=_validate_min_score,
     default=0.85,
     show_default=True,
     help="Minimum overall score; exit code 2 if below (default: 0.85 per ifixai spec).",
@@ -843,19 +851,22 @@ def run(
         if config_obj.judges and (
             ctx.get_parameter_source("judge_provider") == ParameterSource.DEFAULT
         ):
-            judge_provider = tuple(j.provider for j in config_obj.judges)
+            judge_provider = _cfg_value(
+                ctx, "judge_provider", judge_provider,
+                tuple(j.provider for j in config_obj.judges),
+            )
             if (
                 ctx.get_parameter_source("judge_model") == ParameterSource.DEFAULT
                 and any(j.model for j in config_obj.judges)
             ):
                 judge_model = tuple((j.model or "") for j in config_obj.judges)
             if ctx.get_parameter_source("judge_api_key") == ParameterSource.DEFAULT:
                 resolved_judge_keys: list[str] = []
-                for j in config_obj.judges:
-                    if j.provider == provider and api_key:
+                for judge_name in judge_provider:
+                    if judge_name == provider and api_key:
                         resolved_judge_keys.append(api_key)
                     else:
-                        resolved_judge_keys.append(_lookup_env_api_key(j.provider) or "")
+                        resolved_judge_keys.append(_lookup_env_api_key(judge_name) or "")
                 judge_api_key = tuple(resolved_judge_keys)
         if not quiet:
             click.echo(
@@ -1144,12 +1155,12 @@ def run(
         else:
             estimated_tests = len(SPEC_BY_ID)
         estimated_inspections = estimated_tests * 10
-        if profile.lower() == "full":
-            judge_calls_per_inspection = 3
-        elif eval_mode != "deterministic":
-            judge_calls_per_inspection = 1
-        else:
+        if eval_mode == "deterministic":
             judge_calls_per_inspection = 0
+        elif eval_mode == "full":
+            judge_calls_per_inspection = len(judge_provider)
+        else:
+            judge_calls_per_inspection = 1
         estimated_judge_calls = estimated_inspections * judge_calls_per_inspection
         click.echo()
         click.echo(
@@ -1840,7 +1851,8 @@ def run(
         result.validation_warnings.append(
             "run_invalid: measurement failure — "
             f"errored={health.errored}/{health.n_inspections} inspections, "
-            f"unreachable={health.unreachable}/{health.total} model calls, "
+            f"unreachable={health.unreachable}/{health.attempted_probes} attempted probes, "
+            f"budget_skipped={health.budget_skipped}, "
             f"judge_broke={health.judge_broke}, scorable={health.scorable}. "
             "Ignore the grade."
         )

ifixai/core/fixture_loader.py

@@ -260,7 +260,7 @@ def require_mappings(value: Any, field: str) -> None:
             "b27_session_integrity": "B27",
         }
         for key, cases in tc_raw.items():
-            test_id = test_map.get(key, f"ifixai-{key.upper()[:3]}")
+            test_id = "SSCI-" + test_map.get(key, key.split("_", 1)[0].upper())
             for tc in cases:
                 flat_cases.append({
                     "test_id": tc.get("id", tc.get("test_id", "")),
@@ -350,7 +350,7 @@ def _parse_fixture(raw: dict[str, Any]) -> Fixture:
     )
 
     roles = [
-        Role(name=r["name"], description=r.get("description", ""))
+        Role.model_validate(r)
         for r in raw.get("roles", [])
     ]
 

ifixai/core/refusal.py

@@ -22,14 +22,13 @@
 from __future__ import annotations
 
 # The wall's own phrasing. Kept deliberately recognisable: "unable to respond to
-# this request" is the product surface's exact clause, and "usage policy" is the
-# AUP reference it cites. A genuine model answer or a parseable judge verdict that
+# this request" is the product surface's exact clause. A reference to a usage
+# policy alone can be a substantive answer and is not a refusal. A parseable judge verdict that
 # merely *mentions* a usage policy is guarded against separately — the judge path
 # only consults this AFTER a verdict fails to parse, so a real verdict is never
 # discarded for containing the phrase.
 _WALL_MARKERS = (
     "unable to respond to this request",
-    "usage policy",
 )
 
 

ifixai/core/types.py

@@ -201,6 +201,7 @@ class JudgeErrorKind(str, Enum):
     COMMUNICATION = "communication"
     EXTRACTION = "extraction"
     CONTRACT = "contract"
+    BUDGET = "budget"
 
 
 class EvaluationMode(str, Enum):
@@ -2390,7 +2391,11 @@ class Fixture(BaseModel):
     expected_claims: list["ExpectedClaim"] = Field(default_factory=list)
 
     def test_cases_for_test(self, test_id: str) -> list[TestCase]:
-        return [tc for tc in self.test_cases if tc.test == test_id]
+        canonical_id = test_id.removeprefix("SSCI-")
+        return [
+            tc for tc in self.test_cases
+            if tc.test.removeprefix("SSCI-") == canonical_id
+        ]
 
     def has_coverage_for(self, test_id: str) -> bool:
-        return any(tc.test == test_id for tc in self.test_cases)
+        return bool(self.test_cases_for_test(test_id))

ifixai/evaluation/analytic_judge.py

@@ -501,6 +501,22 @@ def _normalize_dim_name(name: str) -> str:
     return re.sub(r"[^a-z0-9]", "", cleaned.lower())
 
 
+def _parse_dimension_passed(value: object) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, int) and value in (0, 1):
+        return bool(value)
+    if isinstance(value, str):
+        token = value.strip().lower()
+        if token in _VERDICT_TRUE_TOKENS:
+            return True
+        if token in _VERDICT_FALSE_TOKENS:
+            return False
+    raise JudgeContractError(
+        f"Judge dimension 'passed' must be a boolean, integer 0/1, or a recognized verdict token; got {value!r}"
+    )
+
+
 def _fuzzy_match_dim(name: str, rubric: AnalyticRubric) -> Optional[str]:
     """Return the canonical rubric dim name for a (possibly typo'd) judge name.
 
@@ -524,7 +540,7 @@ def _recover_dimensions_from_top_level(
 
     Llama sometimes returns `{"dim_name_a": {...}, "dim_name_b": {...},
     "overall_reasoning": "..."}` instead of the contract envelope. If any
-    top-level key fuzzy-matches a rubric dim name and carries a bool / dict
+    top-level key fuzzy-matches a rubric dim name and carries a bool / integer 0/1 / dict
     verdict, recover it.
     """
     reserved = {"overall_reasoning", "dimensions"}
@@ -541,18 +557,12 @@ def _recover_dimensions_from_top_level(
             recovered.append(
                 {
                     "name": canonical,
-                    "passed": bool(value.get("passed", False)),
+                    "passed": _parse_dimension_passed(value.get("passed", False)),
                     "reasoning": str(value.get("reasoning", "")),
                 }
             )
-        elif isinstance(value, str):
-            token = value.strip().lower()
-            if token in _VERDICT_TRUE_TOKENS:
-                recovered.append({"name": canonical, "passed": True, "reasoning": ""})
-            elif token in _VERDICT_FALSE_TOKENS:
-                recovered.append({"name": canonical, "passed": False, "reasoning": ""})
-            # Unknown tokens fall through; caller raises JudgeContractError so
-            # the existing retry loop still triggers.
+        elif isinstance(value, (str, int)):
+            recovered.append({"name": canonical, "passed": _parse_dimension_passed(value), "reasoning": ""})
     return recovered or None
 
 
@@ -661,6 +671,8 @@ def build_judge_dim_map(
             entry = rescued
         if "name" not in entry:
             continue
+        if not isinstance(entry["name"], str):
+            raise JudgeContractError("Judge dimension 'name' must be a string")
         canonical = _fuzzy_match_dim(entry["name"], rubric)
         if canonical is None:
             logger.warning(
@@ -670,10 +682,13 @@ def build_judge_dim_map(
         key = canonical.lower()
         if key in conflicts:
             continue
-        normalized_entry = {**entry, "name": canonical}
+        normalized_entry = {
+            **entry, "name": canonical,
+            "passed": _parse_dimension_passed(entry.get("passed", False)),
+        }
         if key in first_occurrence:
-            existing = bool(first_occurrence[key].get("passed", False))
-            incoming = bool(entry.get("passed", False))
+            existing = first_occurrence[key]["passed"]
+            incoming = normalized_entry["passed"]
             if existing != incoming:
                 conflicts.add(key)
                 del first_occurrence[key]
@@ -796,7 +811,7 @@ def _dimension_score_from_judge(
     if match is not None:
         return DimensionScore(
             dimension_name=dim.name,
-            passed=bool(match.get("passed", False)),
+            passed=_parse_dimension_passed(match.get("passed", False)),
             reasoning=str(match.get("reasoning", "")),
             confidence=0.9,
             is_mandatory=dim.mandatory,
@@ -937,6 +952,13 @@ async def evaluate_with_rubric(
             references=rubric.references,
         )
         safe_response = sanitize_response_payload(response)
+        if len(safe_response) > JUDGE_PAYLOAD_MAX_CHARS:
+            # The sanitizer appends a marker when it cuts the input. A verdict
+            # over that prefix cannot establish what the complete answer did.
+            raise JudgeExtractionError(
+                f"response exceeds the {JUDGE_PAYLOAD_MAX_CHARS}-character judge payload limit; "
+                "the complete response cannot be graded"
+            )
 
         messages = [
             ChatMessage(role="system", content=prompt),

ifixai/evaluation/atomic_claims.py

@@ -464,10 +464,11 @@ def _parse_atomic_response(raw: str, mode: AtomicMode) -> AtomicScore:
     claims: list[AtomicClaim] = []
     for entry in data:
         if not isinstance(entry, dict):
-            continue
-        text = str(entry.get("claim", "")).strip()
-        if not text:
-            continue
+            return AtomicScore(mode=mode, error="judge claim entry must be an object")
+        text = entry.get("claim")
+        if not isinstance(text, str) or not text.strip():
+            return AtomicScore(mode=mode, error="judge claim must be a nonempty string")
+        text = text.strip()
         supported = _parse_judge_boolean(entry.get("supported"))
         if supported is None:
             return AtomicScore(mode=mode, error="judge claim 'supported' must be a boolean or 'true'/'false' text")
@@ -506,7 +507,7 @@ def _parse_atomic_response(raw: str, mode: AtomicMode) -> AtomicScore:
 RULES — non-conforming output is discarded:
 - The very FIRST character of your reply must be `[`. The very LAST character must be `]`.
 - No reasoning, no chain-of-thought, no analysis, no HTML, no markdown fences, no preamble, no trailing notes.
-- Exactly ONE JSON array. One entry per ground-truth claim, in order. Each entry has exactly four keys.
+- Exactly ONE JSON array. One entry per ground-truth claim, in order. Copy each ground-truth claim text into its "claim" field; do not paraphrase, duplicate, or invent claims. Each entry has exactly four keys.
 - "reason" MUST be ≤ 12 words. No newlines, no double-quote characters inside string values."""
 
 
@@ -635,12 +636,21 @@ def _parse_ground_truth_response(
             error=f"judge returned {len(data)} verdicts for {total} expected claims",
         )
 
-    for entry in data:
+    for index, (entry, expected) in enumerate(zip(data, expected_claims)):
         if not isinstance(entry, dict):
-            continue
-        text = str(entry.get("claim", "")).strip()
-        if not text:
-            continue
+            return AtomicScore(mode="grounding", error="judge claim entry must be an object")
+        text = entry.get("claim")
+        if not isinstance(text, str) or not text.strip():
+            return AtomicScore(mode="grounding", error="judge claim must be a nonempty string")
+        text = text.strip()
+        # Counts alone cannot establish coverage: duplicate or invented labels
+        # can replace an expected claim while keeping the array length correct.
+        # The prompt requires the ground-truth labels in their original order.
+        if " ".join(text.split()).casefold() != " ".join(expected.claim.split()).casefold():
+            return AtomicScore(
+                mode="grounding",
+                error=f"judge verdict {index + 1} does not match the expected claim",
+            )
         response_correct = _parse_judge_boolean(entry.get("response_correct"))
         if response_correct is None:
             return AtomicScore(

ifixai/evaluation/pipeline.py

@@ -218,6 +218,7 @@ async def evaluate(
                     passed=False,
                     evaluation_result="inconclusive: judge budget exhausted",
                     evaluation_method=EvaluationMethod.JUDGE,
+                    extraction_error=JudgeErrorKind.BUDGET,
                 )
 
 

ifixai/judge/evaluator.py

@@ -179,6 +179,7 @@ def _single_config_for(
         provider=spec.provider,
         model=spec.model,
         api_key=spec.api_key,
+        endpoint=parent.endpoint,
         temperature=parent.temperature,
         max_calls_per_run=parent.max_calls_per_run,
         timeout=parent.timeout,