feat(core): node-retry budget above the fallback chain (1.S, ADR-0040)

docs/decisions/0038-agentrunner-llm-call-boundary.md

@@ -2,7 +2,14 @@
 
 - **Status**: Accepted
 - **Date**: 2026-06-14
-- **Related**: [ADR-0006](0006-os-keychain-for-api-keys.md), [ADR-0011](0011-internal-llm-abstraction.md), [ADR-0018](0018-desktop-execution-and-rust-egress.md), [ADR-0019](0019-cli-node-keychain-library.md), [ADR-0024](0024-agent-first-entry-point-agentsession.md), [ADR-0025](0025-agent-surface-refines-desktop-scope.md), [ADR-0026](0026-session-export-to-workflow.md), [ADR-0028](0028-workflow-resource-governance.md), [ADR-0029](0029-tool-policy-hardening.md), [ADR-0030](0030-llm-seam-shape-amendment-reasoning-response-format-provider-executed.md), [ADR-0036](0036-run-loop-substrate-event-bus-and-execution-host.md), [ADR-0037](0037-engine-tool-execution-boundary.md), [ADR-0039](0039-same-provider-reasoning-replay.md), [llm-provider-seam.md](../reference/shared-core/llm-provider-seam.md), [run-plan.md](../reference/shared-core/run-plan.md), [sse-event-schema.md](../reference/contracts/sse-event-schema.md), [error-handling.md](../standards/error-handling.md), [security-review.md](../standards/security-review.md), [shared-core-engine.md](../architecture/shared-core-engine.md)
+
+> **Amended by [ADR-0040](0040-node-retry-budget-above-the-chain.md) (2026-06-15):** the detail that fed the
+> node/agent `retry` (`max` **and** `backoff`) into the **primary `FallbackPlanEntry`** is reversed — `retry`
+> is now the engine's *above-chain* node-retry budget, and the primary entry defaults to `maxAttempts: 1` with
+> the chain's own default backoff. Everything else in this ADR stands. (Within-chain same-model retry of the
+> primary, if wanted, is an optional primary chain-entry `max_attempts`.)
+
+- **Related**: [ADR-0006](0006-os-keychain-for-api-keys.md), [ADR-0011](0011-internal-llm-abstraction.md), [ADR-0018](0018-desktop-execution-and-rust-egress.md), [ADR-0019](0019-cli-node-keychain-library.md), [ADR-0024](0024-agent-first-entry-point-agentsession.md), [ADR-0025](0025-agent-surface-refines-desktop-scope.md), [ADR-0026](0026-session-export-to-workflow.md), [ADR-0028](0028-workflow-resource-governance.md), [ADR-0029](0029-tool-policy-hardening.md), [ADR-0030](0030-llm-seam-shape-amendment-reasoning-response-format-provider-executed.md), [ADR-0036](0036-run-loop-substrate-event-bus-and-execution-host.md), [ADR-0037](0037-engine-tool-execution-boundary.md), [ADR-0039](0039-same-provider-reasoning-replay.md), [ADR-0040](0040-node-retry-budget-above-the-chain.md), [llm-provider-seam.md](../reference/shared-core/llm-provider-seam.md), [run-plan.md](../reference/shared-core/run-plan.md), [sse-event-schema.md](../reference/contracts/sse-event-schema.md), [error-handling.md](../standards/error-handling.md), [security-review.md](../standards/security-review.md), [shared-core-engine.md](../architecture/shared-core-engine.md)
 
 ## Context
 

docs/decisions/README.md

@@ -83,6 +83,7 @@ flowchart TD
 | 0037 | [Engine-side tool-execution boundary — the `ToolHost` capability seam, policy/mechanism split, bounded results](0037-engine-tool-execution-boundary.md) | Accepted | 2026-06-13 |
 | 0038 | [AgentRunner LLM-call boundary — host-injected provider resolution, the per-node-execution `FallbackChain`, and the credential discipline](0038-agentrunner-llm-call-boundary.md) | Accepted | 2026-06-14 |
 | 0039 | [Same-provider signed-reasoning replay — a behavioral amendment to ADR-0030](0039-same-provider-reasoning-replay.md) | Accepted | 2026-06-14 |
+| 0040 | [Node-level retry budget above the provider fallback chain (1.S) — amends ADR-0038](0040-node-retry-budget-above-the-chain.md) | Accepted | 2026-06-15 |
 
 ## Creating a new ADR
 

docs/reference/contracts/agent-yaml-spec.md

@@ -34,9 +34,11 @@ memory:                     # optional conversational memory policy
   type: none | window | summary
   window_size: number       # when type = window
 
-retry:                      # transient-error retry on the primary model
-  max: number
+retry:                      # node-retry budget ABOVE the fallback chain (ADR-0040): re-run the whole node on a retryable failure
+  max: number               # total attempts, including the first
   backoff: linear | exponential
+  backoff_ms: number        # optional base delay (ms) the backoff scales; engine-defaulted when omitted
+  retry_on: [error-code]    # optional; one or more of: provider_rate_limit, provider_unavailable, tool_failed, sandbox_error (a non-retryable code is rejected at parse)
 
 fallback_chain:             # ordered alternates tried after the primary is exhausted
   - model: string
@@ -59,15 +61,15 @@ fallback_chain:             # ordered alternates tried after the primary is exha
 | `tools` | no | Tool ids — see [../shared-core/built-in-tools.md](../shared-core/built-in-tools.md). |
 | `mcp_servers` | no | See [../shared-core/mcp-integration.md](../shared-core/mcp-integration.md). |
 | `memory` | no | `none` (default), `window` (last N turns), or `summary` (rolling summary). |
-| `retry` | no | Retry on the *same* model for retryable errors. |
-| `fallback_chain` | no | Switch to a *different* model/provider after retries are exhausted. |
+| `retry` | no | Node-retry budget **above** the fallback chain — re-runs the whole node on a retryable failure, up to `max` total attempts ([ADR-0040](../../decisions/0040-node-retry-budget-above-the-chain.md)). |
+| `fallback_chain` | no | Switch to a *different* model/provider **within an attempt** (the within-chain failover); the node `retry` budget then re-runs the whole chain above it. |
 
 ## Retry vs. fallback
 
 These are two distinct resilience layers:
 
-- **`retry`** handles transient failures (timeouts, rate limits) by re-attempting the **same** model with the configured `backoff`.
-- **`fallback_chain`** handles a model being unavailable or repeatedly failing by moving to the **next** provider/model in the list. Each entry has its own `max_attempts`. The chain is tried in order; the first entry that succeeds produces the node output. This is the mechanism behind the multi-model fallback killer feature.
+- **`retry`** is the engine's **above-chain node-retry budget** ([ADR-0040](../../decisions/0040-node-retry-budget-above-the-chain.md)): on a retryable failure (timeouts, rate limits, a transient tool/sandbox error) it re-dispatches the **whole node** up to `max` total attempts with the configured `backoff` (base `backoff_ms`), optionally restricted to specific codes via `retry_on` (one or more of `provider_rate_limit`, `provider_unavailable`, `tool_failed`, `sandbox_error` — a non-retryable code is rejected at parse, never a silent no-op). It applies to non-agent nodes too (`condition`/`transform`/`merge`). *(Historically `retry` meant within-chain same-model retry; ADR-0040 amended that.)*
+- **`fallback_chain`** handles a model being unavailable or repeatedly failing by moving to the **next** provider/model in the list **within a single attempt**. Each entry has its own `max_attempts`. The chain is tried in order; the first entry that succeeds produces the node output. This is the mechanism behind the multi-model fallback killer feature.
 
 ```yaml
 retry:
@@ -82,7 +84,7 @@ fallback_chain:
     max_attempts: 1
 ```
 
-The resolution order at run time is: primary `model` (with `retry`) → first `fallback_chain` entry (with its `max_attempts`) → next entry → … . The `model`/`provider` pair and the `fallback_chain` are resolved against Relavium's provider-agnostic `LLMProvider` seam: each entry selects a provider adapter and a canonical model id, and the fallback is executed by the `withFallback` runner that sits *outside* the adapters (the adapters stay dumb). Provider-key resolution and cross-provider tool-schema normalization are likewise handled by `@relavium/llm`. The immovable contract for all of this — the request/result/stream types, the normalization rules, and where `fallback_chain` `max_attempts` is enforced — is [../shared-core/llm-provider-seam.md](../shared-core/llm-provider-seam.md); the rationale and supported-model matrix are in [../../architecture/multi-llm-providers.md](../../architecture/multi-llm-providers.md).
+The resolution order **within one attempt** is: primary `model` → first `fallback_chain` entry (with its `max_attempts`) → next entry → … ; the node `retry` budget ([ADR-0040](../../decisions/0040-node-retry-budget-above-the-chain.md)) then re-runs this whole sequence, above the chain, on a retryable failure. The `model`/`provider` pair and the `fallback_chain` are resolved against Relavium's provider-agnostic `LLMProvider` seam: each entry selects a provider adapter and a canonical model id, and the fallback is executed by the `withFallback` runner that sits *outside* the adapters (the adapters stay dumb). Provider-key resolution and cross-provider tool-schema normalization are likewise handled by `@relavium/llm`. The immovable contract for all of this — the request/result/stream types, the normalization rules, and where `fallback_chain` `max_attempts` is enforced — is [../shared-core/llm-provider-seam.md](../shared-core/llm-provider-seam.md); the rationale and supported-model matrix are in [../../architecture/multi-llm-providers.md](../../architecture/multi-llm-providers.md).
 
 ## Example
 

docs/reference/contracts/sse-event-schema.md

@@ -51,6 +51,7 @@ export type RunEvent =
   | NodeCompletedEvent
   | NodeFailedEvent
   | NodeSkippedEvent
+  | NodeRetryingEvent
   | HumanGatePausedEvent
   | HumanGateResumedEvent
   | RunCompletedEvent
@@ -67,14 +68,15 @@ export type RunEvent =
 | `type` | Meaning | Key payload fields |
 | --- | --- | --- |
 | `run:started` | A run began. | `workflowId` (the `workflows.id` **UUID** FK, not the authored slug — [ADR-0022](../../decisions/0022-run-references-workflow-by-uuid.md)), `inputs` (secret-typed inputs **masked** — see [Security](#security-event-payloads-never-carry-secrets)), `executionMode: 'local' \| 'cloud' \| 'managed'` |
-| `node:started` | A node began executing. | `nodeId`, `nodeType` |
+| `node:started` | A node began executing. | `nodeId`, `nodeType`, `attemptNumber?` (1-based; absent ⇒ attempt 1, present + >1 ⇒ a node-retry re-dispatch — 1.S) |
 | `agent:token` | A streaming LLM token from an agent node. | `nodeId`, `token`, `model` |
 | `agent:tool_call` | An agent invoked a tool. | `nodeId`, `model` (the invoking model — so a tool call is attributable across a failover), `toolId`, `toolInput` (sanitized — no secrets), `attemptNumber?` (1-based, matches `cost:updated`) |
 | `agent:tool_result` | A tool returned. | `nodeId`, `toolId`, `success`, `outputSummary` (truncated for UI), `attemptNumber?` |
 | `agent:file_patch_proposed` | An agent proposed a file change (**gated — no write until the user accepts**; e.g. the VS Code inline-diff review). | `nodeId`, `patches: [{ uri, unifiedDiff }]` (≥1 — an empty proposal is meaningless), `attemptNumber?` |
-| `cost:updated` | A node's token cost was tallied (drives the cost waterfall). | `nodeId`, `model`, `inputTokens`, `outputTokens`, `costMicrocents`, `cumulativeCostMicrocents` (integer micro-cents — canonical unit in [llm-provider-seam.md](../shared-core/llm-provider-seam.md#6-usage)), `attemptNumber?` (1-based retry attempt this cost belongs to, so per-attempt cost is reconstructable) |
-| `node:completed` | A node finished successfully. | `nodeId`, `output`, `tokensUsed: {input, output, model?}` (`model` only for LLM nodes), `durationMs`, `selected?` (a `condition`'s chosen target ids — the authoritative branch record checkpoint/resume restores from, 1.R; **may be an empty array** when the condition routes to no branch, dimming all downstream), `attemptNumber?` |
-| `node:failed` | A node failed. | `nodeId`, `error: {code, message, retryable, correlationId?}` (`code` is an [`ErrorCode`](#error-code-taxonomy); `correlationId` is a secret-free id joined to the internal log — ADR-0036) |
+| `cost:updated` | A node's token cost was tallied (drives the cost waterfall). | `nodeId`, `model`, `inputTokens`, `outputTokens`, `costMicrocents`, `cumulativeCostMicrocents` (integer micro-cents — canonical unit in [llm-provider-seam.md](../shared-core/llm-provider-seam.md#6-usage)), `attemptNumber?` (1-based **within-chain** FallbackChain attempt — resets per node-retry re-dispatch; **distinct** from `node:*.attemptNumber`, see the [two attemptNumber families](#two-attemptnumber-families) note) |
+| `node:completed` | A node finished successfully. | `nodeId`, `output`, `tokensUsed: {input, output, model?}` (`model` only for LLM nodes), `durationMs`, `selected?` (a `condition`'s chosen target ids — the authoritative branch record checkpoint/resume restores from, 1.R; **may be an empty array** when the condition routes to no branch, dimming all downstream), `attemptNumber?` (1-based **node-retry** dispatch attempt — 1.S; absent ⇒ attempt 1) |
+| `node:failed` | A node failed (TERMINAL — exactly one per node; emitted when the node-retry budget is exhausted, on a fatal / `retry_on`-excluded failure, **or** when a pending retry is abandoned by a cancel or a sibling abort — see 1.S). | `nodeId`, `error: {code, message, retryable, correlationId?}` (`code` is an [`ErrorCode`](#error-code-taxonomy); `correlationId` is a secret-free id joined to the internal log — ADR-0036), `attemptNumber?` (the last attempt, when a retry budget was spent — 1.S) |
+| `node:retrying` | A retryable node attempt failed and the engine will re-dispatch the whole node (1.S, [ADR-0040](../../decisions/0040-node-retry-budget-above-the-chain.md)) — **non-terminal** (the node continues; `node:failed` is the terminal). | `nodeId`, `attemptNumber` (the attempt that just failed, 1-based), `error: {code, message, retryable}` (the `NodeFailure` shape — **no** `correlationId`; that anchors the terminal failure), `delayMs` (backoff before the next attempt) |
 | `node:skipped` | A node was skip-propagated (never ran). | `nodeId`, `reason: 'branch_not_taken' \| 'upstream_unreachable'` (`branch_not_taken` = a `condition` routed away from it; `upstream_unreachable` = every in-edge is dead because an upstream was skipped/failed). Emitted so the event log is a **complete, replayable** record — checkpoint/resume reconstructs a skipped vertex from it ([run-plan.md](../shared-core/run-plan.md)) and a surface can render the dimmed path instead of the node silently vanishing. |
 | `human_gate:paused` | Execution suspended at a human gate. | `nodeId`, `gateId`, `gateType: 'approval' \| 'input' \| 'review'`, `message`, `assignee?`, `timeoutMs?`, `timeoutAction?: 'approve' \| 'reject'` (on-timeout policy, present only with `timeoutMs`), `expiresAt?` |
 | `human_gate:resumed` | A gate decision was applied; execution continues. | `nodeId`, `decision: 'approved' \| 'rejected' \| 'input_provided'`, `decidedBy`, `payload?` |
@@ -83,6 +85,15 @@ export type RunEvent =
 | `run:failed` | The run failed. | `error: {code, message, retryable, nodeId?, correlationId?}` (`code` is an [`ErrorCode`](#error-code-taxonomy); `nodeId` is the root-cause node; `correlationId` joins to the internal log — ADR-0036), `partialOutputs` |
 | `run:cancelled` | The run was cancelled. | (base only) |
 
+### Two attemptNumber families
+
+`attemptNumber` appears on two **independent** counter families that must not be conflated (1.S, [ADR-0040](../../decisions/0040-node-retry-budget-above-the-chain.md)):
+
+- **Node-retry dispatch attempt** — on `node:started` / `node:completed` / `node:failed` / `node:retrying`. The engine's **above-chain** whole-node re-dispatch index. Absent ⇒ attempt 1; present + >1 ⇒ a re-dispatch (distinguishes "attempt N starting" from a replay).
+- **Within-chain attempt** — on `cost:updated` / `agent:tool_call` / `agent:tool_result` / `agent:file_patch_proposed`. The **within-chain** `FallbackChain` attempt index inside a *single* node dispatch; it **resets to 1 on every node-retry re-dispatch** (a fresh chain runs each time).
+
+The two do **not** join: on a node the budget retried, `node:completed.attemptNumber` may be `2` while the accompanying `cost:updated.attemptNumber` is `1`. To attribute cost to a node-retry attempt, **partition the `sequenceNumber`-ordered stream at each `node:started` / `node:retrying` boundary** — do not key by `(nodeId, attemptNumber)` across families. (Run totals are unaffected: `cost:updated.cumulativeCostMicrocents` is the engine's authoritative running total.)
+
 ### Selected definitions
 
 > These TypeScript shapes are **illustrative**. The enforced, runtime-validated
@@ -107,7 +118,7 @@ export interface CostUpdatedEvent extends BaseEvent {
   outputTokens: number;
   costMicrocents: number;         // integer micro-cents (canonical unit defined in llm-provider-seam.md); this attempt, from Relavium's pricing table (never the provider)
   cumulativeCostMicrocents: number; // integer micro-cents running total for the whole run
-  attemptNumber?: number;         // 1-based retry attempt this cost belongs to (per-attempt cost attribution)
+  attemptNumber?: number;         // 1-based WITHIN-CHAIN attempt; resets per node-retry re-dispatch — distinct from node:*.attemptNumber (see "Two attemptNumber families")
 }
 
 export interface NodeCompletedEvent extends BaseEvent {
@@ -120,7 +131,7 @@ export interface NodeCompletedEvent extends BaseEvent {
   tokensUsed: { input: number; output: number; model?: string };
   durationMs: number;
   selected?: string[];      // a `condition` node only: the immediate target ids it routed to (the live branches); MAY be empty when it routes to no branch (all downstream skip-propagated). The authoritative record checkpoint/resume restores `selectedTargets` from (1.R).
-  attemptNumber?: number;   // 1-based retry attempt this completion belongs to (matches cost:updated)
+  attemptNumber?: number;   // 1-based NODE-RETRY dispatch attempt (1.S); absent ⇒ attempt 1 — distinct from cost:updated.attemptNumber (see "Two attemptNumber families")
 }
 
 export interface NodeSkippedEvent extends BaseEvent {
@@ -129,6 +140,14 @@ export interface NodeSkippedEvent extends BaseEvent {
   reason: 'branch_not_taken' | 'upstream_unreachable';
 }
 
+export interface NodeRetryingEvent extends BaseEvent {
+  type: 'node:retrying';        // 1.S — a retryable attempt failed; the engine will re-dispatch the whole node. NON-TERMINAL.
+  nodeId: string;
+  attemptNumber: number;        // the attempt that just failed (1-based); the next attempt is attemptNumber + 1
+  error: { code: ErrorCode; message: string; retryable: boolean }; // the NodeFailure shape — no correlationId (that anchors the terminal node:failed)
+  delayMs: number;              // backoff before the next attempt
+}
+
 export interface HumanGatePausedEvent extends BaseEvent {
   type: 'human_gate:paused';
   nodeId: string;