fix(ci): read the toolchain directive, run -race per PR, bump otel (agent-os-7tl)

.github/workflows/backend.yml

@@ -11,7 +11,11 @@
 #   failed unit tests can merge on a check that never executed.
 #
 #   This workflow needs no Docker daemon and no registry access, so it stays
-#   green (or goes honestly red) independently of integration.yml.
+#   green (or goes honestly red) independently of integration.yml. That also
+#   makes it the only place a self-contained `//go:build integration` test can
+#   be caught leaking into the default build: such a test compiles and vets
+#   cleanly, and passes on the Docker-equipped integration runner. It fails
+#   here, where there is no daemon.
 #
 # NO BASE-BRANCH FILTER:
 #   integration.yml filters `pull_request` on `branches: [main, 'feat/**']`,
@@ -26,16 +30,20 @@ on:
   pull_request:
     paths:
       - 'backend/**'
+      # The Go version is pinned in two places that must agree — go.mod's
+      # toolchain directive and the Dockerfile's base image. Without this
+      # entry a Dockerfile-only drift triggers no backend job at all, which
+      # is exactly the drift the lockstep requirement exists to prevent.
+      - 'docker/Dockerfile'
       - '.github/workflows/backend.yml'
   push:
     branches:
       - main
     paths:
       - 'backend/**'
+      - 'docker/Dockerfile'
       - '.github/workflows/backend.yml'
-  # Nightly run for the race-detector job below, plus manual one-off runs.
-  schedule:
-    - cron: '0 3 * * *'
+  # Allow manual one-off runs from the Actions tab.
   workflow_dispatch:
 
 jobs:
@@ -44,27 +52,37 @@ jobs:
     runs-on: ubuntu-latest
     # The suite takes well under a minute; this is a hang guard, not a budget.
     timeout-minutes: 10
-    # The nightly schedule exists for the `race` job only.
-    if: github.event_name != 'schedule'
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      # Read the Go version from go.mod so this workflow never drifts from the
-      # toolchain the code is written against. Same step integration.yml uses.
-      - name: Read Go version from go.mod
-        id: goversion
-        run: |
-          VERSION=$(grep '^go ' backend/go.mod | awk '{print $2}')
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Go ${{ steps.goversion.outputs.version }}
-        uses: actions/setup-go@v5
+      # go-version-file reads backend/go.mod, and from v6 setup-go prefers its
+      # `toolchain` directive over the `go` directive — so CI installs exactly
+      # the toolchain the code is pinned to. The v6 pin is load-bearing: v5
+      # reads only the `go` directive, and go.mod carries both `go 1.25.0` and
+      # `toolchain go1.25.12`.
+      #
+      # This replaces a `grep '^go ' backend/go.mod` step that matched that
+      # same wrong line. CI installed 1.25.0 and reached the pinned toolchain
+      # only because GOTOOLCHAIN defaults to `auto`, which makes the go command
+      # silently re-exec into the right version. That worked, but nothing
+      # expressed it and nothing enforced it — GOTOOLCHAIN=local anywhere would
+      # have reverted CI to the older stdlib with no failing check.
+      - name: Set up Go
+        uses: actions/setup-go@v6
         with:
-          go-version: ${{ steps.goversion.outputs.version }}
+          go-version-file: backend/go.mod
           cache-dependency-path: backend/go.sum
 
+      # Record the toolchain that actually compiles. Keep this even though the
+      # setup step now resolves correctly: it is the only line in the log that
+      # states what compiled rather than what was requested, and it is what
+      # caught the v5 behaviour above.
+      - name: Report Go version
+        working-directory: backend
+        run: go version
+
       - name: Build
         working-directory: backend
         run: go build ./...
@@ -79,41 +97,35 @@ jobs:
         working-directory: backend
         run: go test ./... -count=1
 
-  # DECISION — the race detector runs nightly, not per-PR.
-  #
-  # This codebase is exactly the profile the race detector is built for:
+  # This codebase is the profile the race detector is built for:
   # goroutine-per-connection WebSocket handlers, background tickers, a session
-  # map behind a mutex, a connection manager behind an RWMutex. But `-race`
-  # takes ~90s on a developer machine and roughly 4-6 minutes on a two-core
-  # GitHub runner, against ~20s without it. Putting that on every PR would
-  # forfeit the reason this workflow was split out — a fast, always-on check.
+  # map behind a mutex, a connection manager behind an RWMutex.
   #
-  # Races are nondeterministic, so repeated runs find things a single run
-  # misses. A nightly run against main gets that repetition and reports within
-  # a day, which is the right trade for a project of this size. Revisit if a
-  # race ever reaches main and costs more than a day to find.
+  # It runs on every PR, in its own job. Measured on this repo's runners the
+  # -race step takes ~100s against ~25s without, and the job as a whole ~2min
+  # against the unit job's ~40s. Because the two run concurrently, and because
+  # integration.yml takes ~4.5min on the same PR, that puts the race detector
+  # nowhere near the critical path — so there is no reason to defer it to a
+  # nightly and learn about a race up to a day after the commit that caused it.
   race:
-    name: Race detector (nightly)
+    name: Race detector
     runs-on: ubuntu-latest
     timeout-minutes: 20
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Read Go version from go.mod
-        id: goversion
-        run: |
-          VERSION=$(grep '^go ' backend/go.mod | awk '{print $2}')
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Go ${{ steps.goversion.outputs.version }}
-        uses: actions/setup-go@v5
+      - name: Set up Go
+        uses: actions/setup-go@v6
         with:
-          go-version: ${{ steps.goversion.outputs.version }}
+          go-version-file: backend/go.mod
           cache-dependency-path: backend/go.sum
 
+      - name: Report Go version
+        working-directory: backend
+        run: go version
+
       - name: Unit tests with -race
         working-directory: backend
         run: go test ./... -count=1 -race

.github/workflows/integration.yml

@@ -59,17 +59,16 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      # Read the Go version from go.mod so this workflow never drifts.
-      - name: Read Go version from go.mod
-        id: goversion
-        run: |
-          VERSION=$(grep '^go ' backend/go.mod | awk '{print $2}')
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Go ${{ steps.goversion.outputs.version }}
-        uses: actions/setup-go@v5
+      # go-version-file reads backend/go.mod and, from setup-go v6, prefers
+      # its `toolchain` directive over the `go` directive — so this workflow
+      # never drifts from the toolchain the code is pinned to. The v6 pin is
+      # load-bearing; v5 reads only the `go` directive. Replaces a
+      # `grep '^go '` step that matched that same wrong line. Full note in
+      # backend.yml.
+      - name: Set up Go
+        uses: actions/setup-go@v6
         with:
-          go-version: ${{ steps.goversion.outputs.version }}
+          go-version-file: backend/go.mod
           cache-dependency-path: backend/go.sum
 
       # Sanity-check: confirm Docker daemon is up and compose v2 is available.
@@ -108,14 +107,25 @@ jobs:
             ./internal/integrationtest/ \
             ./internal/truth/...
 
-      # Confirm the default (non-tagged) build is still green — guards against
-      # the integration package accidentally leaking build constraints into
-      # the normal build.
+      # Confirm the default (non-tagged) build is still green.
+      #
+      # This catches a `//go:build integration` tag going missing in
+      # ./internal/integrationtest/, because that package's harness is a
+      # non-test file behind the same tag — drop the tag from any test there
+      # and vet reports `undefined: RequireDocker`.
+      #
+      # It does NOT catch a leak in ./internal/truth/, whose integration tests
+      # are self-contained (stdlib plus shelling out to `docker`). Those vet
+      # clean, and they pass here, on a runner that has a daemon. The thing
+      # that catches them is backend.yml's Docker-free `go test ./...`, where
+      # they fail for want of Docker.
       #
       # This used to also run `go test ./... -count=1`. It no longer needs to:
       # backend.yml runs the full unit suite unconditionally, on every PR that
       # touches backend/**, with no Docker dependency. Keeping the run here
-      # only meant the unit signal was hostage to the Docker steps above it.
+      # only meant the unit signal was hostage to the Docker steps above it —
+      # and, per the paragraph above, it would not have caught the truth/ leak
+      # anyway, since a daemon is present.
       - name: Confirm default build unaffected
         working-directory: backend
         run: |

backend/go.mod

@@ -85,10 +85,10 @@ require (
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
-	go.opentelemetry.io/otel v1.41.0 // indirect
+	go.opentelemetry.io/otel v1.42.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.41.0 // indirect
-	go.opentelemetry.io/otel/trace v1.41.0 // indirect
+	go.opentelemetry.io/otel/metric v1.42.0 // indirect
+	go.opentelemetry.io/otel/trace v1.42.0 // indirect
 	go.yaml.in/yaml/v4 v4.0.0-rc.3 // indirect
 	golang.org/x/arch v0.8.0 // indirect
 	golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect

backend/go.sum

@@ -207,20 +207,20 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0=
-go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
-go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40=
-go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
-go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
 go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
 go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
 go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
 go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
-go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
-go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
 go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
 go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=

backend/internal/services/manualremote_test.go

@@ -6,15 +6,32 @@
 //	go test -tags=manualremote -count=1 -v ./internal/services/ -run TestManualRemote
 //
 // Nothing in the 657-test unit suite exercises a real remote, so a green run
-// says nothing about whether the go-git bump broke stack git operations. This
-// drives GitService against github.com/thinkbig1979/capstan over BOTH an HTTPS
-// and an SSH remote.
+// says nothing about whether stack git operations still work. This drives
+// GitService against github.com/thinkbig1979/capstan over BOTH an HTTPS and
+// an SSH remote.
+//
+// READ THIS BEFORE TRUSTING IT AS EVIDENCE ABOUT go-git. It is mostly not.
+// agent-os-r1a — openRepo passes a nil cache to filesystem.NewStorage —
+// panics the go-git path on any repository containing packfiles, which is
+// every cloned repository. GetStatus recovers and falls back to the CLI, so
+// the go-git surface this harness actually reaches is exactly osfs.New,
+// filesystem.NewStorage, git.Open and repo.Head (step 2a). Everything from
+// step 2b down is served by the git CLI, and Pull is CLI by construction
+// (git.go: Pull -> pullCLI). repo.Worktree, worktree.Status, getDivergence,
+// findMergeBase, countCommits and mapCommit are never executed here.
+//
+// So: this is a real end-to-end check of what capstan serves over HTTPS and
+// SSH, and it is NOT a meaningful check of a go-git upgrade. Once r1a is
+// fixed, revisit — and cover a stacks directory containing symlinked
+// subdirectories, since go-git v5.19.x added a symlink-rejecting worktree
+// boundary and go-billy v5.9.x resolves the chroot base through EvalSymlinks.
 package services
 
 import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 )
@@ -56,17 +73,33 @@ func TestManualRemote(t *testing.T) {
 
 			s := &GitService{}
 
-			// 2a. Record which internal path GetStatus resolves to. On a real
-			// clone the go-git path panics inside filesystem.ObjectStorage
-			// because openRepo passes a nil cache to filesystem.NewStorage —
-			// PRE-EXISTING, reproduces identically on main @ go-git v5.17.1,
-			// tracked separately. Every object in a fresh clone lives in a
-			// packfile, and the packfile reader is the only thing that touches
-			// the cache, which is why locally-built test repos never hit it.
+			// 2a. The only genuine go-git assertions in this file. openRepo is
+			// osfs.New + filesystem.NewStorage + git.Open; repo.Head() is the
+			// reference lookup. Both must work against a real clone, and both
+			// are reached before agent-os-r1a's nil-cache panic fires further
+			// down in CommitObject.
+			repo, err := s.openRepo(local)
+			if err != nil {
+				t.Fatalf("openRepo (go-git) failed on a real clone: %v", err)
+			}
+			headRef, err := repo.Head()
+			if err != nil {
+				t.Fatalf("repo.Head (go-git) failed on a real clone: %v", err)
+			}
+			if want := runGit(t, local, "rev-parse", "HEAD"); headRef.Hash().String() != want {
+				t.Errorf("go-git Head = %s, git CLI says %s", headRef.Hash(), want)
+			}
+			if headRef.Name().Short() != "main" {
+				t.Errorf("go-git Head ref = %q, want main", headRef.Name().Short())
+			}
+			t.Logf("go-git reached: openRepo + Head OK at %s", headRef.Hash())
+
+			// Record where it stops. Every assertion below 2a is served by the
+			// git CLI because of this — see the file header.
 			if _, err := s.getStatusGoGit(local); err != nil {
-				t.Logf("NOTE: go-git path unavailable, GetStatus falls back to CLI: %v", err)
+				t.Logf("NOTE: go-git path dies here, GetStatus falls back to CLI: %v", err)
 			} else {
-				t.Logf("NOTE: go-git path succeeded")
+				t.Logf("NOTE: go-git path succeeded — r1a may be fixed; revisit this file")
 			}
 
 			// 2b. STATUS through the public API — what capstan actually serves.
@@ -96,7 +129,9 @@ func TestManualRemote(t *testing.T) {
 			t.Logf("status: branch=%s commit=%s author=%q remote=%s",
 				st.Branch, st.Commit.Short, st.Commit.Author, st.RemoteURL)
 
-			// 3. STATUS reports a dirty worktree (go-git worktree.Status).
+			// 3. STATUS reports a dirty worktree. Served by `git status
+			// --porcelain` in getStatusCLI, NOT by go-git's worktree.Status —
+			// see the note at 2a. go-git's worktree is never reached.
 			if err := os.WriteFile(filepath.Join(local, "vkr-scratch.txt"), []byte("dirty\n"), 0o644); err != nil {
 				t.Fatal(err)
 			}
@@ -125,11 +160,16 @@ func TestManualRemote(t *testing.T) {
 			if err != nil {
 				t.Fatalf("GetStatus after rewind failed: %v", err)
 			}
-			// Rewinding one first-parent step off a merge commit puts the
-			// clone >=1 commits behind (the merge plus what it merged), so
-			// assert the direction, not an exact count.
-			if behindSt.Behind < 1 || behindSt.Ahead != 0 {
-				t.Errorf("after rewind: ahead=%d behind=%d, want ahead=0 behind>=1", behindSt.Ahead, behindSt.Behind)
+			// Derive the expected count rather than hardcoding it. Rewinding
+			// one first-parent step off a merge commit leaves the clone more
+			// than one commit behind, and this clones a live moving remote, so
+			// the repo shape is not under the test's control.
+			wantBehind := runGit(t, local, "rev-list", "--count", "HEAD..origin/main")
+			if got := strconv.Itoa(behindSt.Behind); got != wantBehind {
+				t.Errorf("after rewind: behind=%s, git rev-list says %s", got, wantBehind)
+			}
+			if behindSt.Ahead != 0 {
+				t.Errorf("after rewind: ahead=%d, want 0", behindSt.Ahead)
 			}
 			t.Logf("status after rewind: ahead=%d behind=%d", behindSt.Ahead, behindSt.Behind)