feat(agents): stamp the project scope into runContext; pool keys prefer it over the mount

Decision 1 option (b): the runner's keep-alive pool scopes sessions by project id.
Until now that scope came only from the mount-sign response, so a run without a
durable mount could never park. The trustworthy project id now rides runContext:
the service stamps it server-side from the request's authenticated baggage (never
from a caller-supplied wire field), and the runner PREFERS runContext.project.id,
falling back to the mount scope. The no-scope-no-park safety rule is unchanged.

Claude-Session: https://claude.ai/code/session_01AumZJ9xRd4XYNHThqy4rTv

Author: mmabrouk

docs/design/agent-workflows/documentation/running-the-agent.md

@@ -178,8 +178,12 @@ sections).
   (a `0` would re-disable auto-stop and reintroduce the leak).
 - `AGENTA_RUNNER_SESSION_KEEPALIVE`. Gates session keep-alive: after a turn ends, the runner
   parks the live harness session and continues it on the next matching message in the same
-  conversation, instead of cold-replaying the transcript. Default off. Local sandbox only;
-  requires mount signing (no mount scope means the session never parks). Design:
+  conversation, instead of cold-replaying the transcript. Default off. Local sandbox only. The
+  parked-session pool is keyed by project scope so a live session never crosses a project
+  boundary. That scope comes from the project id the service stamps into the run context
+  (`runContext.project.id`, derived server-side from the request's auth, never from a
+  caller-supplied field); the mount's owning project id is the fallback when a run carries no
+  stamped project. A run with no project scope from either source never parks. Design:
   `docs/design/agent-workflows/projects/session-keepalive/plan.md`.
 - `AGENTA_RUNNER_SESSION_TTL_MS`. How long an idle parked session lives before it is
   destroyed. Default `60000`.

sdks/python/agenta/sdk/agents/__init__.py

@@ -70,6 +70,7 @@
     PermissionMode,
     PiAgentTemplate,
     RunContext,
+    RunContextProject,
     RunContextReference,
     RunContextRun,
     RunContextTrace,
@@ -175,6 +176,7 @@
     "ui_message_stream",
     "TraceContext",
     "RunContext",
+    "RunContextProject",
     "RunContextReference",
     "RunContextRun",
     "RunContextWorkflow",

sdks/python/agenta/sdk/agents/dtos.py

@@ -466,6 +466,19 @@ class RunContextRun(BaseModel):
     kind: Optional[str] = None
 
 
+class RunContextProject(BaseModel):
+    """The run's owning project identity inside ``runContext`` (so ``$ctx.project.id`` is
+    addressable, mirroring how the workflow references nest).
+
+    ``id`` is the SERVER-derived project id: the service stamps it from its own request state (the
+    authenticated OTel baggage), NEVER from anything the caller sends. It is the project scope the
+    runner trusts for session keep-alive — the runner keys its parked-session pool on it so a live
+    session can never be resumed across a project boundary. See :class:`RuntimeAuthContext` for the
+    same "from the request state, never the request body" discipline."""
+
+    id: Optional[str] = None
+
+
 class RunContext(BaseModel):
     """The run's own context, delivered on ``/run`` and refreshed per turn (direct-call tools,
     Phase 3a; see ``projects/direct-call-tools/run-context.md``).
@@ -481,9 +494,14 @@ class RunContext(BaseModel):
     id is NOT carried here — it rides the top-level ``sessionId`` field, and the runner owns the
     live id across turns; duplicating it in run context would only let it go stale. ``to_wire``
     emits only the sub-objects/fields that are set, so a run with no identity yields an empty blob
-    (and the serializer omits the key entirely)."""
+    (and the serializer omits the key entirely).
+
+    ``project`` carries the run's owning project id, stamped server-side (see
+    :class:`RunContextProject`). It is the trustworthy project scope for session keep-alive: the
+    runner prefers it over the mount-derived scope when keying its parked-session pool."""
 
     run: Optional[RunContextRun] = None
+    project: Optional[RunContextProject] = None
     workflow: Optional[RunContextWorkflow] = None
     trace: Optional[RunContextTrace] = None
 
@@ -497,6 +515,14 @@ def to_wire(self) -> Dict[str, Any]:
             }
             if run:
                 out["run"] = run
+        if self.project is not None:
+            project = {
+                key: value
+                for key, value in self.project.model_dump().items()
+                if value is not None
+            }
+            if project:
+                out["project"] = project
         if self.workflow is not None:
             workflow: Dict[str, Any] = {}
             for entity in ("artifact", "variant", "revision"):

sdks/python/agenta/sdk/agents/tracing.py

@@ -21,6 +21,7 @@
 
 from agenta.sdk.agents.dtos import (
     RunContext,
+    RunContextProject,
     RunContextReference,
     RunContextTrace,
     RunContextWorkflow,
@@ -180,6 +181,23 @@ def _run_context_trace() -> Optional[RunContextTrace]:
     )
 
 
+def _run_context_project() -> Optional[RunContextProject]:
+    """The run's owning project id, read from the SERVER-derived request context.
+
+    The source is the authenticated OTel ``baggage`` on :class:`TracingContext` — the same
+    ``project_id`` the auth middleware scopes its permission check on (``request.state.otel``),
+    NEVER a value the caller put in the request body. Stamping it here keeps the trust boundary at
+    the service: the runner then trusts it because the service is trusted (mirrors
+    :class:`~agenta.sdk.agents.connections.models.RuntimeAuthContext.project_id`). The runner uses
+    it as the preferred project scope for session keep-alive. Best-effort: no baggage / no
+    ``project_id`` returns ``None`` and the field is simply omitted."""
+    baggage = TracingContext.get().baggage or {}
+    project_id = baggage.get("project_id")
+    if not project_id:
+        return None
+    return RunContextProject(id=str(project_id))
+
+
 def run_context() -> Optional[RunContext]:
     """Capture the run's own context for tool bindings and run-kind propagation.
 
@@ -191,9 +209,15 @@ def run_context() -> Optional[RunContext]:
     Best-effort: any failure (or an entirely empty context) returns ``None`` so the run proceeds and
     the ``runContext`` key is simply omitted.
 
-    The workflow and the trace are captured as INDEPENDENT failure domains: a failure reading the
-    workflow references must not drop an otherwise-valid ``trace`` (and vice versa), so a trace-only
-    run still ships ``runContext.trace``."""
+    The project, the workflow, and the trace are captured as INDEPENDENT failure domains: a failure
+    reading one must not drop the others, so a run still ships whichever parts it holds (a
+    trace-only run still ships ``runContext.trace``; a run with only a project scope still ships
+    ``runContext.project``)."""
+    project = None
+    try:
+        project = _run_context_project()
+    except Exception:  # pylint: disable=broad-except
+        log.warning("agent: failed to capture run-context project", exc_info=True)
     workflow = None
     try:
         workflow = _run_context_workflow()
@@ -204,9 +228,9 @@ def run_context() -> Optional[RunContext]:
         trace = _run_context_trace()
     except Exception:  # pylint: disable=broad-except
         log.warning("agent: failed to capture run-context trace", exc_info=True)
-    if workflow is None and trace is None:
+    if project is None and workflow is None and trace is None:
         return None
-    return RunContext(workflow=workflow, trace=trace)
+    return RunContext(project=project, workflow=workflow, trace=trace)
 
 
 def record_usage(usage: Optional[Dict[str, Any]]) -> None:

sdks/python/agenta/sdk/agents/wire_models.py

@@ -202,6 +202,14 @@ class WireRunContextRun(_WireModel):
     kind: Optional[str] = None
 
 
+class WireRunContextProject(_WireModel):
+    """The run's owning project identity inside ``runContext`` (mirrors ``RunContextProject``).
+    ``id`` is the SERVER-stamped project id — never a caller-supplied value. The keys stay
+    snake_case on purpose — see ``WireRunContext``."""
+
+    id: Optional[str] = None
+
+
 class WireRunContext(_WireModel):
     """The run's own context, delivered on ``/run`` and refreshed per turn (direct-call tools,
     Phase 3a; mirrors ``RunContext.to_wire``).
@@ -216,6 +224,7 @@ class WireRunContext(_WireModel):
     ``runContext`` on the request."""
 
     run: Optional[WireRunContextRun] = None
+    project: Optional[WireRunContextProject] = None
     workflow: Optional[WireRunContextWorkflow] = None
     trace: Optional[WireRunContextTrace] = None
 

sdks/python/oss/tests/pytest/unit/agents/golden/run_request.pi_core.json

@@ -31,6 +31,7 @@
   },
   "runContext": {
     "run": {"kind": "test"},
+    "project": {"id": "proj_abc"},
     "workflow": {
       "artifact": {"id": "wf_abc"},
       "variant": {"id": "var_abc", "slug": "weather-agent"},

sdks/python/oss/tests/pytest/unit/agents/test_tracing.py

@@ -8,7 +8,11 @@
 
 from types import SimpleNamespace
 
-from agenta.sdk.agents import RunContextTrace, RunContextWorkflow
+from agenta.sdk.agents import (
+    RunContextProject,
+    RunContextTrace,
+    RunContextWorkflow,
+)
 
 from agenta.sdk.agents import tracing
 
@@ -50,8 +54,10 @@ def boom():
     assert ctx.workflow == RunContextWorkflow(is_draft=True)
 
 
-def test_run_context_none_when_both_empty(monkeypatch):
-    # No workflow identity and no trace -> no run context at all (the key is omitted on the wire).
+def test_run_context_none_when_all_empty(monkeypatch):
+    # No project, no workflow identity, and no trace -> no run context at all (the key is omitted
+    # on the wire).
+    monkeypatch.setattr(tracing, "_run_context_project", lambda: None)
     monkeypatch.setattr(tracing, "_run_context_workflow", lambda: None)
     monkeypatch.setattr(tracing, "_run_context_trace", lambda: None)
     assert tracing.run_context() is None
@@ -84,3 +90,59 @@ def test_run_context_workflow_normalizes_application_references(monkeypatch):
     assert workflow.revision.id == "revision-id"
     assert workflow.revision.version == "v2"
     assert workflow.is_draft is False
+
+
+def test_run_context_project_stamped_from_server_baggage(monkeypatch):
+    # The owning project id is read from the SERVER-derived request context (the authenticated
+    # OTel baggage on TracingContext), never from anything the caller sends. This is the source
+    # the runner trusts to scope its keep-alive pool.
+    monkeypatch.setattr(
+        tracing.TracingContext,
+        "get",
+        lambda: SimpleNamespace(baggage={"project_id": "proj-42"}),
+    )
+
+    project = tracing._run_context_project()
+
+    assert project == RunContextProject(id="proj-42")
+
+
+def test_run_context_project_none_without_baggage(monkeypatch):
+    # No baggage / no project_id in the request state -> no project scope; the field is omitted
+    # and the runner falls back to the mount-derived scope.
+    monkeypatch.setattr(
+        tracing.TracingContext,
+        "get",
+        lambda: SimpleNamespace(baggage=None),
+    )
+    assert tracing._run_context_project() is None
+
+    monkeypatch.setattr(
+        tracing.TracingContext,
+        "get",
+        lambda: SimpleNamespace(baggage={"other": "x"}),
+    )
+    assert tracing._run_context_project() is None
+
+
+def test_run_context_keeps_project_when_workflow_and_trace_fail(monkeypatch):
+    # The project scope is its own failure domain: a run that holds only a project id (no workflow,
+    # no trace) still ships `runContext.project` so the runner can key its keep-alive pool on it.
+    def boom():
+        raise RuntimeError("unavailable")
+
+    monkeypatch.setattr(
+        tracing,
+        "_run_context_project",
+        lambda: RunContextProject(id="proj-42"),
+    )
+    monkeypatch.setattr(tracing, "_run_context_workflow", boom)
+    monkeypatch.setattr(tracing, "_run_context_trace", boom)
+
+    ctx = tracing.run_context()
+    assert ctx is not None
+    assert ctx.project == RunContextProject(id="proj-42")
+    assert ctx.workflow is None
+    assert ctx.trace is None
+    # The project rides the wire under the snake_case `project.id` binding namespace.
+    assert ctx.to_wire() == {"project": {"id": "proj-42"}}

sdks/python/oss/tests/pytest/unit/agents/test_wire_contract.py

@@ -28,6 +28,7 @@
     PiAgentTemplate,
     ResolvedConnection,
     RunContext,
+    RunContextProject,
     RunContextReference,
     RunContextRun,
     RunContextTrace,
@@ -149,6 +150,9 @@ def _pi_payload():
         # not run context.
         run_context=RunContext(
             run=RunContextRun(kind="test"),
+            # The owning project id, stamped server-side (F5): the trustworthy scope the runner
+            # prefers when keying its keep-alive pool, over the mount-derived fallback.
+            project=RunContextProject(id="proj_abc"),
             workflow=RunContextWorkflow(
                 artifact=RunContextReference(id="wf_abc"),
                 variant=RunContextReference(id="var_abc", slug="weather-agent"),
@@ -267,6 +271,9 @@ def test_request_to_wire_pi_matches_golden(golden):
     # by `to_wire`. The conversation id is NOT here — it rides the top-level `sessionId`.
     assert payload["runContext"] == {
         "run": {"kind": "test"},
+        # The server-stamped project scope (F5), snake_case inner key `project.id` — the
+        # `$ctx.project.id` binding namespace and the runner's preferred keep-alive pool scope.
+        "project": {"id": "proj_abc"},
         "workflow": {
             "artifact": {"id": "wf_abc"},
             "variant": {"id": "var_abc", "slug": "weather-agent"},

services/runner/src/engines/sandbox_agent.ts

@@ -431,7 +431,8 @@ export interface SessionEnvironment {
   runAgentDir: string | undefined;
   otlpAuthFilePath: string | undefined;
   mountCreds: MountCredentials | null;
-  /** The mount's owning project id (keep-alive pool key scope); undefined when there is no mount. */
+  /** The mount's owning project id (keep-alive pool key FALLBACK scope, preferred is
+   * `runContext.project.id`); undefined when there is no mount. */
   mountProjectId?: string;
   // Mutable teardown/turn state shared across acquire, runTurn, and destroy.
   sessionDestroyRequested: boolean;
@@ -471,12 +472,14 @@ export type AcquireEnvironmentResult =
 
 /**
  * Sign the session's durable mount up front so keep-alive can build a pool key (the mount's
- * owning `projectId`) and credential epoch without acquiring the whole environment. Returns
+ * owning `projectId`, the FALLBACK project scope when the run carries no service-stamped
+ * `runContext.project.id`) and credential epoch without acquiring the whole environment. Returns
  * exactly what the sign yielded: `null` when there is no session/credential to sign with, or
  * the sign returned no usable mount (store unconfigured, 503, ephemeral fallback). The caller
  * threads the result — null included — into `acquireEnvironment` as `presignedMount`, so the
- * mount is signed exactly once per run on every path; a null result additionally means there is
- * NO safe project key and the request must never park.
+ * mount is signed exactly once per run on every path. A null result no longer forces a cold run
+ * on its own: the request still parks when the run context supplied a project scope, and only
+ * skips parking when NEITHER source yields one (`poolKeyFor` returns null).
  */
 export async function resolveKeepaliveMount(
   request: AgentRunRequest,

services/runner/src/engines/sandbox_agent/mount.ts

@@ -33,9 +33,10 @@ export interface MountCredentials {
   expiresAt?: string;
   /**
    * The mount's owning project id, surfaced from the sign response's `mount` object. It is the
-   * only project scope the runner can trust for this request (the /run wire carries no project
-   * id today), so session keep-alive keys its pool on `<projectId>:<sessionId>`. Absent when the
-   * response omitted the mount object; keep-alive then refuses to park (no safe key source).
+   * FALLBACK project scope for session keep-alive: the pool prefers the service-stamped
+   * `runContext.project.id` and falls back to this mount scope when the run carries no stamped
+   * project (see `poolKeyFor`). Absent when the response omitted the mount object; keep-alive
+   * then parks only if the run context supplied a scope, and refuses to park when neither does.
    */
   projectId?: string;
 }