fix: preserve completed signal-exit classification

Author: seancdavis

src/cli.ts

@@ -535,7 +535,7 @@ program
         process.exit(1);
       }
       jobFilter = manifest.results
-        .filter((r) => r.exitCode !== 0 || r.error)
+        .filter((r) => r.failed ?? (r.exitCode !== 0 || r.error))
         .map((r) => ({ scenarioKey: r.scenarioKey, agentName: r.agentName }));
       if (jobFilter.length === 0) {
         process.stderr.write(`\n  No failed jobs in report ${manifest.reportId}. Nothing to retry.\n\n`);

src/docs-site/src/pages/cli.astro

@@ -90,9 +90,9 @@ import DocsLayout from "../layouts/DocsLayout.astro";
       <div class="cli-flag">
         <span class="cli-flag-name"><code>--failed [reportId]</code></span>
         <span class="cli-flag-desc">
-          Re-run only the failed scenario/agent pairs from a previous report
-          (default: <code>latest</code>). Pairs no longer in the config are
-          silently dropped. Mutually exclusive with <code>--scenario</code> and
+          Re-run only the scenario/agent pairs marked failed in a previous report
+          (default: <code>latest</code>). Pairs no longer in the config are silently
+          dropped. Mutually exclusive with <code>--scenario</code> and
           <code>--agent</code>.
         </span>
       </div>

src/docs-site/src/pages/running.astro

@@ -122,6 +122,11 @@ export default createAgentAdapter<{ stdout: string }>({
     agents). Each variant appears as a separate row in the CLI output and a separate entry in
     reports, identified by its <code>@</code>-suffixed key (e.g., <code>create-post@with-mcp</code>).
   </p>
+  <p>
+    Report manifest entries include a <code>failed</code> boolean computed from the full run output before
+    transcripts are stripped from <code>report.json</code>. This preserves the correct status for agents that
+    return a final result even when their process exits non-zero during cleanup.
+  </p>
   <p>
     See <a href="/scenarios#variants">Writing Scenarios &rarr; Variants</a> for the full field
     reference and examples.

src/report-ui/src/scripts/render.ts

@@ -365,7 +365,7 @@ function renderScenarioHeaderRow(group: ScenarioGroup, _startIndex: number, hasS
 
 function renderAgentRow(entry: ResultEntry, index: number, hasScores: boolean, scenarioKey: string): string {
   const s = entry.score;
-  const isFailed = entry.exitCode !== 0 || !!entry.error;
+  const isFailed = entry.failed ?? (entry.exitCode !== 0 || !!entry.error);
   const errorBtn = entry.error
     ? `<button class="error-btn" data-error-index="${index}" title="${escapeHtml(friendlyError(entry.error))}">!</button>`
     : "";

src/report-ui/src/scripts/types.ts

@@ -28,6 +28,7 @@ export interface ResultEntry {
   agentName: string;
   durationMs: number;
   exitCode: number;
+  failed?: boolean;
   tokenUsage?: TokenUsage;
   totalCostUsd?: number;
   score?: ScoreResult;

src/reports/writer.ts

@@ -1,7 +1,7 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import type { RunOutput, RunResult } from "../types/output.js";
-import { isScoredResult } from "../types/output.js";
+import { isFailedRun, isScoredResult } from "../types/output.js";
 import type { ScoredOutput, ScoredRunResult, SparseIndex } from "../types/scoring.js";
 import type { ReportManifest, ReportResultEntry } from "../types/report.js";
 import { generateReportHtml } from "./html.js";
@@ -151,6 +151,7 @@ function buildResultEntry(result: RunResult | ScoredRunResult, relPath: string):
     agentName: result.agentName,
     durationMs: result.output.metadata.durationMs,
     exitCode: result.output.metadata.exitCode,
+    failed: isFailedRun(result.output),
     file: relPath,
   };
 

src/scoring/index.ts

@@ -63,7 +63,7 @@ export async function scoreRunResult(result: RunResult, options?: ScoringOptions
   // get perfect-score defaults in env/service/agent because nothing was audited.
   if (isFailedRun(result.output)) {
     const score = buildZeroScore(result, weights, sparseIndex.lines.length > 0 ? sparseIndex : undefined, judgeAgent);
-    options?.onProgress?.(result.scenarioKey, result.agentName, "done");
+    options?.onProgress?.(result.scenarioKey, result.agentName, "failed");
     result.output.transcriptAnalysis = toTranscriptAnalysis(normalized);
     return {
       scenarioKey: result.scenarioKey,

src/types/output.ts

@@ -141,6 +141,6 @@ export function formatError(err: unknown): string {
  */
 export function isFailedRun(output: AgentOutput): boolean {
   const { exitCode, error } = output.metadata;
-  if (output.result) return Boolean(error);
+  if (output.result !== null) return Boolean(error);
   return exitCode !== 0 || Boolean(error);
 }

src/types/report.ts

@@ -24,6 +24,8 @@ export interface ReportResultEntry {
   agentName: string;
   durationMs: number;
   exitCode: number;
+  /** Failure classification computed before the manifest is flattened. */
+  failed?: boolean;
   tokenUsage?: TokenUsage;
   totalCostUsd?: number;
   score?: ScoreResult;

src/types/scoring.ts

@@ -208,7 +208,7 @@ export interface ScoringOptions {
   weights?: ScoringWeights;
   logger?: Logger;
   /** Called when scoring starts/finishes for a result. */
-  onProgress?: (scenarioKey: string, agentName: string, phase: "start" | "done") => void;
+  onProgress?: (scenarioKey: string, agentName: string, phase: "start" | "done" | "failed") => void;
   /** Report directory for writing raw data before judges run. */
   reportDir?: string;
   /**