From 00d1f8d5a350ea8d7782a327fa6b0a0f5971d46a Mon Sep 17 00:00:00 2001
From: Mahmoud Mabrouk <mahmoud@agenta.ai>
Date: Tue, 14 Jul 2026 18:15:37 +0200
Subject: [PATCH] fix(hosting): bundle the SeaweedFS store in the gh.local and
 gh.ssl compose variants

PR #5313 added the bundled store and flipped the gh env default to http://seaweedfs:8333, but only added the service to docker-compose.gh.yml. The gh.local (build-from-source) and gh.ssl variants reuse the same gh env file, so they inherited the endpoint default with no such service on the network. Add the identical seaweedfs service, api depends_on, and seaweed-data volume to oss gh.local, oss gh.ssl, and ee gh.local, plus a multi-replica JWT note in the gh env examples.

Claude-Session: https://claude.ai/code/session_01XhENr63WL9npkKrJGnzDc1
---
 .../ee/docker-compose.gh.local.yml            | 62 +++++++++++++++++++
 hosting/docker-compose/ee/env.ee.gh.example   |  3 +
 .../oss/docker-compose.gh.local.yml           | 62 +++++++++++++++++++
 .../oss/docker-compose.gh.ssl.yml             | 62 +++++++++++++++++++
 hosting/docker-compose/oss/env.oss.gh.example |  3 +
 5 files changed, 192 insertions(+)

diff --git a/hosting/docker-compose/ee/docker-compose.gh.local.yml b/hosting/docker-compose/ee/docker-compose.gh.local.yml
index 5b094fb9c9..5f044cc576 100644
--- a/hosting/docker-compose/ee/docker-compose.gh.local.yml
+++ b/hosting/docker-compose/ee/docker-compose.gh.local.yml
@@ -68,6 +68,8 @@ services:
                 condition: service_healthy
             redis-durable:
                 condition: service_healthy
+            seaweedfs:
+                condition: service_healthy
         # === LABELS =============================================== #
         labels:
             - "traefik.http.routers.api.rule=PathPrefix(`/api/`)"
@@ -394,6 +396,65 @@ services:
             retries: 5
             start_period: 5s
 
+    seaweedfs:
+        # === IMAGE ================================================ #
+        # The bundled durable object store for session/agent mounts. Without it, runner mount
+        # signing returns 503 and agent file writes are silently lost. Pinned (not :latest) so the
+        # IAM/STS subsystem stays on a known-good build — it has regressed across releases.
+        image: chrislusf/seaweedfs:4.37
+        # === EXECUTION ============================================ #
+        # ONLY FOR SEAWEEDFS (the bundled store). Real S3/R2/MinIO ship their own STS/IAM and ignore
+        # all of this. Two configs, both generated from env (no committed files):
+        #  - s3.json: the master identity only (admin; the API holds these creds, never the runner).
+        #  - iam.json: the ADVANCED IAM config — the only path SeaweedFS authorizes STS credentials.
+        #    An OIDC provider points at the API's self-served JWKS; the API mints a short-lived
+        #    RS256 web-identity token and calls AssumeRoleWithWebIdentity to assume `agenta-store`.
+        entrypoint:
+            - sh
+            - -c
+            - |
+              cat > /etc/seaweedfs/s3.json <<EOF
+              {"identities":[{"name":"agenta","credentials":[{"accessKey":"$${AGENTA_STORE_ACCESS_KEY}","secretKey":"$${AGENTA_STORE_SECRET_KEY}"}],"actions":["Admin","Read","Write","List","Tagging"]}]}
+              EOF
+              cat > /etc/seaweedfs/iam.json <<EOF
+              {"sts":{"tokenDuration":"1h","maxSessionLength":"12h","issuer":"seaweedfs-sts","signingKey":"$${AGENTA_STORE_SIGNING_KEY}"},"providers":[{"name":"agenta","type":"oidc","enabled":true,"config":{"issuer":"$${AGENTA_STORE_JWT_ISSUER}","clientId":"agenta-store","jwksUri":"$${AGENTA_STORE_JWT_ISSUER}/.well-known/jwks.json"}}],"policies":[{"name":"store-rw","document":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::$${AGENTA_STORE_BUCKET}","arn:aws:s3:::$${AGENTA_STORE_BUCKET}/*"]}]}}],"roles":[{"roleName":"agenta-store","roleArn":"arn:aws:iam::role/agenta-store","attachedPolicies":["store-rw"],"trustPolicy":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"agenta"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}}]}
+              EOF
+              exec weed server -dir=/data -ip=seaweedfs -volume.max=64 -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -s3.iam.config=/etc/seaweedfs/iam.json
+        # === CONFIGURATION ======================================== #
+        env_file:
+            - ${ENV_FILE:-./.env.ee.gh}
+        environment:
+            # The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must
+            # be present in this service's env (defaults here are for the bundled store, the same way
+            # supertokens carries its own connection URI). Keys come from the env file — secrets never
+            # get a baked-in default. Override the whole trio via the env file to use S3/R2/MinIO.
+            AGENTA_STORE_ACCESS_KEY: ${AGENTA_STORE_ACCESS_KEY}
+            AGENTA_STORE_SECRET_KEY: ${AGENTA_STORE_SECRET_KEY}
+            AGENTA_STORE_BUCKET: ${AGENTA_STORE_BUCKET:-agenta-store}
+            AGENTA_STORE_SIGNING_KEY: ${AGENTA_STORE_SIGNING_KEY}
+            AGENTA_STORE_JWT_ISSUER: ${AGENTA_STORE_JWT_ISSUER:-http://api:8000}
+        # === STORAGE ============================================== #
+        volumes:
+            - seaweed-data:/data
+        # === NETWORK ============================================== #
+        networks:
+            - agenta-ee-gh-network
+        # Loopback-only by default (never expose the store to a public IP); override AGENTA_STORE_PORT
+        # to change. In-network services reach it directly at seaweedfs:8333, no publish needed.
+        ports:
+            - "${AGENTA_STORE_PORT:-127.0.0.1:8333}:8333"
+        # === LABELS =============================================== #
+        labels:
+            - "traefik.enable=false"
+        # === LIFECYCLE ============================================ #
+        restart: always
+        healthcheck:
+            test: ["CMD-SHELL", "curl -sf http://localhost:9333/cluster/healthz >/dev/null || exit 1"]
+            interval: 5s
+            timeout: 5s
+            retries: 30
+            start_period: 5s
+
     traefik:
         # === IMAGE ================================================ #
         image: traefik:2
@@ -504,3 +565,4 @@ volumes:
     postgres-data:
     redis-volatile-data:
     redis-durable-data:
+    seaweed-data:
diff --git a/hosting/docker-compose/ee/env.ee.gh.example b/hosting/docker-compose/ee/env.ee.gh.example
index f436232ae2..9a77fdd534 100644
--- a/hosting/docker-compose/ee/env.ee.gh.example
+++ b/hosting/docker-compose/ee/env.ee.gh.example
@@ -329,6 +329,9 @@ AGENTA_STORE_SECRET_KEY=replace-me
 # AGENTA_STORE_NAMESPACE=
 # AGENTA_STORE_SIGNING_KEY=
 # AGENTA_STORE_JWT_ISSUER=http://api:8000
+# The bundled store's web-identity path mints an ephemeral keypair per api process, so a SINGLE
+# api replica works with this unset. If you scale the api past one replica, set the SAME PEM on
+# every replica or STS token minting fails intermittently (SeaweedFS caches one JWKS).
 # AGENTA_STORE_JWT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----"
 # AGENTA_WORKER_STREAMS / AGENTA_WORKER_QUEUES: worker topology selectors —
 # set inline per-service in compose, not here; see docs/designs/workers-sprawl/specs.md
diff --git a/hosting/docker-compose/oss/docker-compose.gh.local.yml b/hosting/docker-compose/oss/docker-compose.gh.local.yml
index 2f12342c7e..7b6c750373 100644
--- a/hosting/docker-compose/oss/docker-compose.gh.local.yml
+++ b/hosting/docker-compose/oss/docker-compose.gh.local.yml
@@ -66,6 +66,8 @@ services:
                 condition: service_healthy
             redis-durable:
                 condition: service_healthy
+            seaweedfs:
+                condition: service_healthy
         # === LABELS =============================================== #
         labels:
             - "traefik.http.routers.api.rule=PathPrefix(`/api/`)"
@@ -392,6 +394,65 @@ services:
             retries: 5
             start_period: 5s
 
+    seaweedfs:
+        # === IMAGE ================================================ #
+        # The bundled durable object store for session/agent mounts. Without it, runner mount
+        # signing returns 503 and agent file writes are silently lost. Pinned (not :latest) so the
+        # IAM/STS subsystem stays on a known-good build — it has regressed across releases.
+        image: chrislusf/seaweedfs:4.37
+        # === EXECUTION ============================================ #
+        # ONLY FOR SEAWEEDFS (the bundled store). Real S3/R2/MinIO ship their own STS/IAM and ignore
+        # all of this. Two configs, both generated from env (no committed files):
+        #  - s3.json: the master identity only (admin; the API holds these creds, never the runner).
+        #  - iam.json: the ADVANCED IAM config — the only path SeaweedFS authorizes STS credentials.
+        #    An OIDC provider points at the API's self-served JWKS; the API mints a short-lived
+        #    RS256 web-identity token and calls AssumeRoleWithWebIdentity to assume `agenta-store`.
+        entrypoint:
+            - sh
+            - -c
+            - |
+              cat > /etc/seaweedfs/s3.json <<EOF
+              {"identities":[{"name":"agenta","credentials":[{"accessKey":"$${AGENTA_STORE_ACCESS_KEY}","secretKey":"$${AGENTA_STORE_SECRET_KEY}"}],"actions":["Admin","Read","Write","List","Tagging"]}]}
+              EOF
+              cat > /etc/seaweedfs/iam.json <<EOF
+              {"sts":{"tokenDuration":"1h","maxSessionLength":"12h","issuer":"seaweedfs-sts","signingKey":"$${AGENTA_STORE_SIGNING_KEY}"},"providers":[{"name":"agenta","type":"oidc","enabled":true,"config":{"issuer":"$${AGENTA_STORE_JWT_ISSUER}","clientId":"agenta-store","jwksUri":"$${AGENTA_STORE_JWT_ISSUER}/.well-known/jwks.json"}}],"policies":[{"name":"store-rw","document":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::$${AGENTA_STORE_BUCKET}","arn:aws:s3:::$${AGENTA_STORE_BUCKET}/*"]}]}}],"roles":[{"roleName":"agenta-store","roleArn":"arn:aws:iam::role/agenta-store","attachedPolicies":["store-rw"],"trustPolicy":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"agenta"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}}]}
+              EOF
+              exec weed server -dir=/data -ip=seaweedfs -volume.max=64 -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -s3.iam.config=/etc/seaweedfs/iam.json
+        # === CONFIGURATION ======================================== #
+        env_file:
+            - ${ENV_FILE:-./.env.oss.gh}
+        environment:
+            # The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must
+            # be present in this service's env (defaults here are for the bundled store, the same way
+            # supertokens carries its own connection URI). Keys come from the env file — secrets never
+            # get a baked-in default. Override the whole trio via the env file to use S3/R2/MinIO.
+            AGENTA_STORE_ACCESS_KEY: ${AGENTA_STORE_ACCESS_KEY}
+            AGENTA_STORE_SECRET_KEY: ${AGENTA_STORE_SECRET_KEY}
+            AGENTA_STORE_BUCKET: ${AGENTA_STORE_BUCKET:-agenta-store}
+            AGENTA_STORE_SIGNING_KEY: ${AGENTA_STORE_SIGNING_KEY}
+            AGENTA_STORE_JWT_ISSUER: ${AGENTA_STORE_JWT_ISSUER:-http://api:8000}
+        # === STORAGE ============================================== #
+        volumes:
+            - seaweed-data:/data
+        # === NETWORK ============================================== #
+        networks:
+            - agenta-oss-gh-network
+        # Loopback-only by default (never expose the store to a public IP); override AGENTA_STORE_PORT
+        # to change. In-network services reach it directly at seaweedfs:8333, no publish needed.
+        ports:
+            - "${AGENTA_STORE_PORT:-127.0.0.1:8333}:8333"
+        # === LABELS =============================================== #
+        labels:
+            - "traefik.enable=false"
+        # === LIFECYCLE ============================================ #
+        restart: always
+        healthcheck:
+            test: ["CMD-SHELL", "curl -sf http://localhost:9333/cluster/healthz >/dev/null || exit 1"]
+            interval: 5s
+            timeout: 5s
+            retries: 30
+            start_period: 5s
+
     traefik:
         # === ACTIVATION =========================================== #
         profiles:
@@ -507,3 +568,4 @@ volumes:
     postgres-data:
     redis-volatile-data:
     redis-durable-data:
+    seaweed-data:
diff --git a/hosting/docker-compose/oss/docker-compose.gh.ssl.yml b/hosting/docker-compose/oss/docker-compose.gh.ssl.yml
index 39d689c441..d5b0bca86f 100644
--- a/hosting/docker-compose/oss/docker-compose.gh.ssl.yml
+++ b/hosting/docker-compose/oss/docker-compose.gh.ssl.yml
@@ -71,6 +71,8 @@ services:
                 condition: service_healthy
             redis-durable:
                 condition: service_healthy
+            seaweedfs:
+                condition: service_healthy
         # === LABELS =============================================== #
         labels:
             - "traefik.http.routers.api.rule=Host(`${TRAEFIK_DOMAIN}`) && PathPrefix(`/api/`)"
@@ -416,6 +418,65 @@ services:
             retries: 5
             start_period: 5s
 
+    seaweedfs:
+        # === IMAGE ================================================ #
+        # The bundled durable object store for session/agent mounts. Without it, runner mount
+        # signing returns 503 and agent file writes are silently lost. Pinned (not :latest) so the
+        # IAM/STS subsystem stays on a known-good build — it has regressed across releases.
+        image: chrislusf/seaweedfs:4.37
+        # === EXECUTION ============================================ #
+        # ONLY FOR SEAWEEDFS (the bundled store). Real S3/R2/MinIO ship their own STS/IAM and ignore
+        # all of this. Two configs, both generated from env (no committed files):
+        #  - s3.json: the master identity only (admin; the API holds these creds, never the runner).
+        #  - iam.json: the ADVANCED IAM config — the only path SeaweedFS authorizes STS credentials.
+        #    An OIDC provider points at the API's self-served JWKS; the API mints a short-lived
+        #    RS256 web-identity token and calls AssumeRoleWithWebIdentity to assume `agenta-store`.
+        entrypoint:
+            - sh
+            - -c
+            - |
+              cat > /etc/seaweedfs/s3.json <<EOF
+              {"identities":[{"name":"agenta","credentials":[{"accessKey":"$${AGENTA_STORE_ACCESS_KEY}","secretKey":"$${AGENTA_STORE_SECRET_KEY}"}],"actions":["Admin","Read","Write","List","Tagging"]}]}
+              EOF
+              cat > /etc/seaweedfs/iam.json <<EOF
+              {"sts":{"tokenDuration":"1h","maxSessionLength":"12h","issuer":"seaweedfs-sts","signingKey":"$${AGENTA_STORE_SIGNING_KEY}"},"providers":[{"name":"agenta","type":"oidc","enabled":true,"config":{"issuer":"$${AGENTA_STORE_JWT_ISSUER}","clientId":"agenta-store","jwksUri":"$${AGENTA_STORE_JWT_ISSUER}/.well-known/jwks.json"}}],"policies":[{"name":"store-rw","document":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::$${AGENTA_STORE_BUCKET}","arn:aws:s3:::$${AGENTA_STORE_BUCKET}/*"]}]}}],"roles":[{"roleName":"agenta-store","roleArn":"arn:aws:iam::role/agenta-store","attachedPolicies":["store-rw"],"trustPolicy":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"agenta"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}}]}
+              EOF
+              exec weed server -dir=/data -ip=seaweedfs -volume.max=64 -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -s3.iam.config=/etc/seaweedfs/iam.json
+        # === CONFIGURATION ======================================== #
+        env_file:
+            - ${ENV_FILE:-./.env.oss.gh}
+        environment:
+            # The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must
+            # be present in this service's env (defaults here are for the bundled store, the same way
+            # supertokens carries its own connection URI). Keys come from the env file — secrets never
+            # get a baked-in default. Override the whole trio via the env file to use S3/R2/MinIO.
+            AGENTA_STORE_ACCESS_KEY: ${AGENTA_STORE_ACCESS_KEY}
+            AGENTA_STORE_SECRET_KEY: ${AGENTA_STORE_SECRET_KEY}
+            AGENTA_STORE_BUCKET: ${AGENTA_STORE_BUCKET:-agenta-store}
+            AGENTA_STORE_SIGNING_KEY: ${AGENTA_STORE_SIGNING_KEY}
+            AGENTA_STORE_JWT_ISSUER: ${AGENTA_STORE_JWT_ISSUER:-http://api:8000}
+        # === STORAGE ============================================== #
+        volumes:
+            - seaweed-data:/data
+        # === NETWORK ============================================== #
+        networks:
+            - agenta-gh-ssl-network
+        # Loopback-only by default (never expose the store to a public IP); override AGENTA_STORE_PORT
+        # to change. In-network services reach it directly at seaweedfs:8333, no publish needed.
+        ports:
+            - "${AGENTA_STORE_PORT:-127.0.0.1:8333}:8333"
+        # === LABELS =============================================== #
+        labels:
+            - "traefik.enable=false"
+        # === LIFECYCLE ============================================ #
+        restart: always
+        healthcheck:
+            test: ["CMD-SHELL", "curl -sf http://localhost:9333/cluster/healthz >/dev/null || exit 1"]
+            interval: 5s
+            timeout: 5s
+            retries: 30
+            start_period: 5s
+
     traefik:
         # === IMAGE ================================================ #
         image: traefik:2
@@ -496,3 +557,4 @@ volumes:
     postgres-data:
     redis-volatile-data:
     redis-durable-data:
+    seaweed-data:
diff --git a/hosting/docker-compose/oss/env.oss.gh.example b/hosting/docker-compose/oss/env.oss.gh.example
index 0c122b61f9..1085301e66 100644
--- a/hosting/docker-compose/oss/env.oss.gh.example
+++ b/hosting/docker-compose/oss/env.oss.gh.example
@@ -329,6 +329,9 @@ AGENTA_STORE_SECRET_KEY=replace-me
 # AGENTA_STORE_NAMESPACE=
 # AGENTA_STORE_SIGNING_KEY=
 # AGENTA_STORE_JWT_ISSUER=http://api:8000
+# The bundled store's web-identity path mints an ephemeral keypair per api process, so a SINGLE
+# api replica works with this unset. If you scale the api past one replica, set the SAME PEM on
+# every replica or STS token minting fails intermittently (SeaweedFS caches one JWKS).
 # AGENTA_STORE_JWT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----"
 # AGENTA_WORKER_STREAMS / AGENTA_WORKER_QUEUES: worker topology selectors —
 # set inline per-service in compose, not here; see docs/designs/workers-sprawl/specs.md