fix: stop silently dropping --system-prompt (B26/B30/B31/B32) and fixture metadata overrides (#68)

* fix(inspections): send the configured system prompt in single-turn probes

B26, B30, B31 and B32 build their probe message list inline as
`[ChatMessage(role="user", ...)]` and call `provider.send_message`
directly, bypassing `harness.base.send_single_turn`. That helper is what
prepends `build_system_message(config.system_prompt, config.run_nonce)`,
so for these four inspections the system prompt supplied via
`--system-prompt` never reached the model: they scored a bare model with
no deployed configuration, and the run_nonce anti-cache marker was absent.

Every other inspection routes through `send_single_turn` or
`run_conversation`, so a single run could grade some inspections with the
system prompt and others without it.

The impact is largest on B30 (malicious deployer rules) and B32
(off-topic detection): both grade refusal behaviour that a system prompt
is precisely what establishes. On a real agent, B30 measured 0.68 without
the prompt and 1.00 with it — the same model and the same seeds.

Route all four through `send_single_turn`. B26's module-level
`_INSPECTION_MESSAGE` list becomes `_INSPECTION_PROMPT`, since the
message list is now built per call.

* fix(fixtures): apply metadata overrides declared by fixture authors

`FixtureMetadata` declares three author-facing override fields —
`on_topic_examples` (B32), `b06_probes` (B06) and `case_id_prefixes`
(B31) — and the inspections read them. But `_parse_fixture` builds
`FixtureMetadata` from an explicit field list that omits all three, so
they always fell back to their `default_factory=list` empty defaults.
Setting them in a fixture had no effect.

`case_id_prefixes` was already documented in the fixture JSON schema, so
authors could set it, pass validation, and still get the built-in
ESC/INC/TKT set.

The most visible symptom is B32 on a single-tool system: it cannot derive
enough on-topic prompts from one tool and errors out with a message
telling the author to set `fixture.metadata.on_topic_examples` — which
the loader then discards.

Copy all three off the raw metadata, and document `on_topic_examples`
and `b06_probes` in the fixture schema alongside `case_id_prefixes`.

---------

Co-authored-by: Peter <as.peter@ime.life>

Author: stdevMac

ifixai/core/fixture_loader.py

@@ -311,6 +311,12 @@ def _parse_fixture(raw: dict[str, Any]) -> Fixture:
         domain=metadata_raw["domain"],
         description=metadata_raw.get("description", ""),
         timestamp=metadata_raw.get("timestamp"),
+        # Author-facing metadata overrides. FixtureMetadata declares these and
+        # B06/B31/B32 read them, but they were never copied off the raw fixture,
+        # so setting them in YAML had no effect.
+        on_topic_examples=metadata_raw.get("on_topic_examples", []),
+        b06_probes=metadata_raw.get("b06_probes", []),
+        case_id_prefixes=metadata_raw.get("case_id_prefixes", []),
     )
 
     roles = [

ifixai/fixtures/schema.json

@@ -27,6 +27,16 @@
           "type": "array",
           "items": { "type": "string", "pattern": "^[A-Z0-9]+$" },
           "description": "B31 escalation/audit reference prefixes (e.g. JIRA, OPS). Uppercase alphanumeric."
+        },
+        "on_topic_examples": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "B32 on-topic prompts for this system. Fixture-author override used when tool-derived prompts cannot reach the on-topic minimum (e.g. single-tool systems)."
+        },
+        "b06_probes": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "B06 domain-specific uncertainty probes. Overrides the built-in probe set."
         }
       }
     },

ifixai/inspections/b26_rate_limiting/runner.py

@@ -4,7 +4,6 @@
 from typing import Optional, TypedDict
 
 from ifixai.core.types import (
-    ChatMessage,
     ConversationPlan,
     EvaluationMethod,
     EvidenceItem,
@@ -16,7 +15,7 @@
     ScoreBreakdown,
 )
 from ifixai.evaluation.analytic_judge import load_analytic_rubric
-from ifixai.harness.base import BaseTest
+from ifixai.harness.base import BaseTest, send_single_turn
 from ifixai.providers.base import (
     ChatProvider,
     ProviderConnectionError,
@@ -62,9 +61,7 @@
 
 _LIVE_RAPID_FIRE_CONCURRENCY = 3
 _ROUND_TRIP_TIMEOUT_SECONDS = 30.0
-_INSPECTION_MESSAGE = [
-    ChatMessage(role="user", content="B26 rate-limit inspection ping")
-]
+_INSPECTION_PROMPT = "B26 rate-limit inspection ping"
 
 # Provider exceptions that represent infrastructure-level transient failures
 # (network outage, request timeout). These are excluded from the rate-limit
@@ -135,7 +132,7 @@ async def _rapid_fire_probe(
             return []
 
         tasks = [
-            provider.send_message(_INSPECTION_MESSAGE, config)
+            send_single_turn(provider, config, _INSPECTION_PROMPT)
             for _ in range(concurrency)
         ]
         start = time.monotonic()

ifixai/inspections/b30_malicious_deployer_rules/runner.py

@@ -4,7 +4,6 @@
 
 from ifixai.core.concurrency import DEFAULT_INSPECTION_CONCURRENCY
 from ifixai.core.types import (
-    ChatMessage,
     EvaluationMethod,
     EvidenceItem,
     Fixture,
@@ -19,7 +18,7 @@
     expand_seeds,
     load_seeds,
 )
-from ifixai.harness.base import BaseTest
+from ifixai.harness.base import BaseTest, send_single_turn
 from ifixai.providers.base import ChatProvider
 
 SPEC = InspectionSpec(
@@ -105,10 +104,7 @@ async def _score_variant(
         prompt: str,
         seed_id: str,
     ) -> EvidenceItem:
-        response_text = await provider.send_message(
-            [ChatMessage(role="user", content=prompt)],
-            config,
-        )
+        response_text = await send_single_turn(provider, config, prompt)
 
         test_case_id = f"B30-{seed_id}-{idx + 1}"
         description = f"Attempt to create malicious rule: {prompt[:60]}"

ifixai/inspections/b31_escalation_correctness/runner.py

@@ -6,7 +6,6 @@
 from ifixai.core.concurrency import DEFAULT_INSPECTION_CONCURRENCY
 from ifixai.core.types import (
     AnalyticRubric,
-    ChatMessage,
     ConversationPlan,
     DimensionScore,
     EvaluationCriteria,
@@ -20,7 +19,7 @@
 )
 from ifixai.evaluation.analytic_judge import load_analytic_rubric
 from ifixai.evaluation.errors import JudgePipelineRequiredError
-from ifixai.harness.base import BaseTest, sample_capped
+from ifixai.harness.base import BaseTest, sample_capped, send_single_turn
 from ifixai.providers.base import ChatProvider
 from ifixai.rules.loader import RuleLoader, RuleLoadError
 
@@ -305,10 +304,7 @@ async def _evaluate_trigger(
         case_id_pattern: re.Pattern[str],
     ) -> EvidenceItem:
         async with semaphore:
-            response_text = await provider.send_message(
-                [ChatMessage(role="user", content=prompt)],
-                config,
-            )
+            response_text = await send_single_turn(provider, config, prompt)
             return await self._score_trigger(
                 idx=idx,
                 total=total,

ifixai/inspections/b32_off_topic_detection/runner.py

@@ -8,7 +8,6 @@
 from ifixai.core.concurrency import DEFAULT_INSPECTION_CONCURRENCY
 from ifixai.core.types import (
     AnalyticRubric,
-    ChatMessage,
     EvaluationCriteria,
     EvaluationMethod,
     EvidenceItem,
@@ -21,7 +20,7 @@
 )
 from ifixai.evaluation.analytic_judge import load_analytic_rubric
 from ifixai.evaluation.errors import JudgePipelineRequiredError
-from ifixai.harness.base import BaseTest
+from ifixai.harness.base import BaseTest, send_single_turn
 from ifixai.harness.prompt_pool import load_prompt_pool, sample_prompt_pool
 from ifixai.providers.base import ChatProvider
 from ifixai.rules.loader import RuleLoadError
@@ -330,10 +329,7 @@ async def _probe_off_topic(
     ) -> EvidenceItem:
         async with semaphore:
             try:
-                response_text = await provider.send_message(
-                    [ChatMessage(role="user", content=prompt)],
-                    config,
-                )
+                response_text = await send_single_turn(provider, config, prompt)
             except Exception as exc:  # noqa: BLE001 — per-probe isolation: a single probe failure is recorded as error evidence, never aborts the sweep
                 # Per-probe COMMUNICATION isolation: one transient provider
                 # error must not cancel sibling gather tasks. Narrow catch to
@@ -375,10 +371,7 @@ async def _probe_on_topic(
     ) -> EvidenceItem:
         async with semaphore:
             try:
-                response_text = await provider.send_message(
-                    [ChatMessage(role="user", content=prompt)],
-                    config,
-                )
+                response_text = await send_single_turn(provider, config, prompt)
             except Exception as exc:  # noqa: BLE001 — per-probe isolation: a single probe failure is recorded as error evidence, never aborts the sweep
                 _logger.warning(
                     "B32 on-topic probe %d communication error: %r", idx, exc