gc: native roots beyond aarch64-macOS — x86-64, iOS, iPadOS, tvOS

.github/workflows/gc-native-roots.yml

@@ -50,9 +50,11 @@
 # wild address, and no address validation after it can help. The wording here
 # before was a guess, and it pointed at the wrong fix.
 #
-# The compiler now refuses that target outright (#7324) rather than emitting a
+# x86-64 was refused outright until #7349 taught the runtime to derive an
+# SP-relative base from the CFA; it is a first-class arm of the matrix now. The
+# old note said (#7324) that a run there would only
 # binary that crashes under collection, so an x86-64 run of this matrix would
-# test nothing but the refusal — which is what `statepoints-refuse-x86` is for.
+# test nothing but the refusal — no longer true, and that job is gone.
 #
 # The same walk is unsound on aarch64 **Linux** too, where it is merely the
 # non-default path: #7333.
@@ -75,9 +77,9 @@
 # CLAUDE.md's GC knob kill-policy: an arm exercising the non-default state, or
 # delete the mode.
 #
-#   PERRY_GC_SAFEPOINT_ONLY  -> native-roots-aarch64, "safepoint-only" steps
-#   PERRY_STACKMAP_WALKER    -> native-roots-aarch64, "both non-default walkers"
-#   PERRY_RS4GC              -> native-roots-rs4gc-aarch64
+#   PERRY_GC_SAFEPOINT_ONLY  -> native-roots-rs4gc, "safepoint-only" steps
+#   PERRY_STACKMAP_WALKER    -> native-roots-rs4gc, "both non-default walkers"
+#   PERRY_RS4GC              -> native-roots-rs4gc
 #   PERRY_STATEPOINT_REPORT  -> deleted. It was a second spelling of
 #                               `--statepoint-report`; the flag is now the only
 #                               entry point and the "fails closed" step is its
@@ -95,8 +97,24 @@ on:
   workflow_dispatch:
 
 jobs:
-  native-roots-rs4gc-aarch64:
-    runs-on: macos-14
+  # Both host shapes Perry supports for native roots, on one job. macOS covers
+  # aarch64 + Mach-O; ubuntu covers x86-64 + ELF — and ELF is where every
+  # object-format bug in this design surfaced (SHF_GNU_RETAIN, SHF_WRITE, the
+  # Mach-O underscore convention in eh_walker). ARM64 Linux would cover the
+  # fourth corner, but those runners queue for hours here, and its two
+  # components are each covered above.
+  native-roots-rs4gc:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: macos-14
+            arch: aarch64
+            format: Mach-O
+          - os: ubuntu-latest
+            arch: x86-64
+            format: ELF
+    runs-on: ${{ matrix.os }}
     # 120, not 90: the in-process step below builds a second time with the
     # llvm-inprocess feature, which cargo cannot share with the build above.
     timeout-minutes: 120
@@ -127,8 +145,21 @@ jobs:
           # `nocreateundeforpoison`, Homebrew opt 22 feeding Apple clang, which
           # is the pairing Perry's own independent discovery picks by default on
           # a Mac. Anyone enabling this knob hits that; pin both here.
-          brew list llvm >/dev/null 2>&1 || brew install llvm
-          llvm_bin="$(brew --prefix llvm)/bin"
+          if [ "$RUNNER_OS" = "macOS" ]; then
+            brew list llvm >/dev/null 2>&1 || brew install llvm
+            llvm_bin="$(brew --prefix llvm)/bin"
+          else
+            # Ubuntu ships a matched opt+clang pair; take the newest present,
+            # and install one only if the image has none.
+            llvm_bin="$(dirname "$(command -v opt || true)")"
+            if [ ! -x "$llvm_bin/opt" ]; then
+              llvm_bin="$(ls -d /usr/lib/llvm-*/bin 2>/dev/null | sort -V | tail -1 || true)"
+            fi
+            if [ ! -x "$llvm_bin/opt" ]; then
+              sudo apt-get update -qq && sudo apt-get install -y -qq llvm clang
+              llvm_bin="$(dirname "$(command -v opt)")"
+            fi
+          fi
           if [ ! -x "$llvm_bin/opt" ] || [ ! -x "$llvm_bin/clang" ]; then
             echo "::error::no matched opt+clang pair under $llvm_bin — RS4GC cannot run, and silently skipping it is exactly the gate that cannot fail"
             exit 1
@@ -147,10 +178,17 @@ jobs:
             name=$(basename "$probe" .ts)
             node --expose-gc --experimental-strip-types "$probe" > "/tmp/rs4gc-$name.oracle"
             PERRY_RS4GC=1 ./target/perry-dev/perry "$probe" -o "/tmp/rs4gc-$name"
-            otool -l "/tmp/rs4gc-$name" | grep -q "sectname __perry_gcmap" \
-              || { echo "::error::$name has no __perry_gcmap section — RS4GC produced no native root map"; exit 1; }
-            otool -l "/tmp/rs4gc-$name" | grep -q "sectname __llvm_stackmaps" \
-              && { echo "::error::$name still carries __llvm_stackmaps — the compact rewrite did not run"; exit 1; }
+            if [ "$RUNNER_OS" = "macOS" ]; then
+              otool -l "/tmp/rs4gc-$name" | grep -q "sectname __perry_gcmap" \
+                || { echo "::error::$name has no __perry_gcmap section — RS4GC produced no native root map"; exit 1; }
+              otool -l "/tmp/rs4gc-$name" | grep -q "sectname __llvm_stackmaps" \
+                && { echo "::error::$name still carries __llvm_stackmaps — the compact rewrite did not run"; exit 1; }
+            else
+              readelf -S "/tmp/rs4gc-$name" | grep -q "\.perry_gcmap" \
+                || { echo "::error::$name has no .perry_gcmap section — RS4GC produced no native root map"; exit 1; }
+              readelf -S "/tmp/rs4gc-$name" | grep -q "\.llvm_stackmaps" \
+                && { echo "::error::$name still carries .llvm_stackmaps — the compact rewrite did not run"; exit 1; }
+            fi
             PERRY_RS4GC=1 PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 \
             PERRY_GC_HEAP_LIMIT=8 PERRY_GC_INCREMENTAL=0 PERRY_CONSERVATIVE_STACK_SCAN=off \
               "/tmp/rs4gc-$name" > "/tmp/rs4gc-$name.out" 2> "/tmp/rs4gc-$name.err"
@@ -190,7 +228,10 @@ jobs:
       # PERRY_LLVM_* variables UNSET -- otherwise it proves nothing the steps
       # above have not already proven.
       - name: RS4GC works on a stock toolchain via the in-process backend
-        if: ${{ !cancelled() }}
+        # macOS only: the assertions below read Mach-O section names. The point
+        # of the step is the stock-toolchain path, which the ELF arm covers by
+        # using the system LLVM in the first place.
+        if: ${{ !cancelled() && runner.os == 'macOS' }}
         run: |
           set -euo pipefail
           export PERRY_RUNTIME_DIR="$PWD/target/perry-dev"
@@ -239,49 +280,8 @@ jobs:
   # REFUSAL (never a silently rootless binary), and goes red the day x86-64
   # starts working, which is the prompt to widen the aarch64 matrix above (#7321).
   # Deliberately cheap: one probe, no runtime, no oracle.
-  statepoints-refuse-x86:
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
-        with:
-          shared-key: gc-native-roots-x86
-      - name: Build compiler and static runtime (perry-dev profile)
-        run: |
-          export RUSTFLAGS="-C force-frame-pointers=yes -C force-unwind-tables=yes"
-          cargo build --profile perry-dev -p perry -p perry-runtime-static -p perry-stdlib-static
-      - name: Statepoints must refuse, not silently drop roots
-        run: |
-          set -uo pipefail
-          export PERRY_RUNTIME_DIR="$PWD/target/perry-dev"
-          export PERRY_NO_AUTO_OPTIMIZE=1
-          probe=benchmarks/gc_ratchet/probes/01_nursery_churn.ts
-          set +e
-          PERRY_RS4GC=1 ./target/perry-dev/perry "$probe" -o /tmp/x86-probe \
-            > /tmp/x86.out 2> /tmp/x86.err
-          rc=$?
-          set -e
-          tail -40 /tmp/x86.out /tmp/x86.err || true
-          if [ "$rc" -eq 0 ]; then
-            echo "::error::native GC roots now compile on x86-64. That is good news and this job is the wrong shape for it: add the x86-64 host to native-roots-rs4gc-aarch64 (rename it) and delete this job."
-            exit 1
-          fi
-          # Non-zero for the RIGHT reason. Any old failure (missing clang, a
-          # broken checkout) would also be non-zero, and a job green on an
-          # unrelated error is the hazard this whole workflow is about.
-          if ! grep -qiE "stack map|compact-map|gc roots would be invisible" /tmp/x86.out /tmp/x86.err; then
-            echo "::error::statepoint compilation failed on x86-64, but not with the compact-map refusal this job asserts. Read the output above: either the refusal message changed, or something unrelated is broken."
-            exit 1
-          fi
-          echo "x86-64: statepoint compilation refuses, as expected, with the compact-map message."
-
-  # Fan-in, mirroring `conformance-smoke-complete` in test.yml: ONE context for
-  # branch protection to require, so adding an arm never needs a protection edit
-  # and a red arm cannot hide behind a green sibling.
   gc-native-roots-complete:
-    needs: [native-roots-rs4gc-aarch64, statepoints-refuse-x86]
+    needs: [native-roots-rs4gc]
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 5
@@ -291,9 +291,8 @@ jobs:
           set -euo pipefail
           failed=0
           for arm in \
-            "native-roots-aarch64=${{ needs.native-roots-aarch64.result }}" \
-            "native-roots-rs4gc-aarch64=${{ needs.native-roots-rs4gc-aarch64.result }}" \
-            "statepoints-refuse-x86=${{ needs.statepoints-refuse-x86.result }}"; do
+            "native-roots-rs4gc=${{ needs.native-roots-rs4gc.result }}" \
+            ; do
             echo "$arm"
             case "$arm" in
               *=success) ;;

changelog.d/7349-apple-platforms.md

@@ -0,0 +1,30 @@
+### Native GC roots reach iOS, iPadOS and tvOS
+
+iOS and iPadOS are aarch64 + Mach-O — the same shape as macOS, which already
+worked. They did not work, and the reason was gating rather than anything
+architectural: the Mach-O stack-map loader, the unwinder module, the fp-chain
+walker and `stack_top` were each `#[cfg(target_os = "macos")]`.
+
+On any other Apple platform that sent the runtime to the no-section stub, so
+`loaded_stack_map_section()` returned `None`, the index came out empty, and the
+collector ran **with no native roots at all** — silently, on the platforms that
+are hardest to debug. The compiler would happily emit the map; nothing read it.
+
+All four gates are now the same predicate: 64-bit Apple, or Linux.
+`pthread_get_stackaddr_np` is Apple-wide, not macOS-only, and the `mach2`
+dependency was widened to match the code that uses it — declaring it for fewer
+targets than the loader compiles on is how this stayed hidden.
+
+**watchOS is refused, deliberately.** `arm64_32` has 32-bit pointers while the
+map stores function addresses as `u64` and the runtime does `usize` arithmetic
+on them. The compiler rejects the target before emitting a map nothing can
+read, and the check is ordered before the `arm64` prefix test so it actually
+fires.
+
+Verified by compiling `perry-runtime` for each target: `aarch64-apple-ios`,
+`aarch64-apple-ios-sim` and `aarch64-apple-tvos` all build. That is what found
+the hole — `stack_top` did not exist on iOS, so the build failed outright
+rather than quietly selecting the stub. `aarch64-apple-visionos` still fails in
+the third-party `psm` build script, unrelated to this change.
+
+Running on a device or simulator is the verification this does not yet have.

changelog.d/7350-statepoints-x86-64.md

@@ -0,0 +1,51 @@
+### Native GC roots work on x86-64
+
+`PERRY_RS4GC=1` was refused on anything but aarch64 (#7324). The refusal was
+correct — the runtime could not resolve x86-64 frame bases, and the collector
+segfaulted rather than reporting anything — but the cause was one unsupported
+call, not anything architectural.
+
+On x86-64 every root is `Indirect [RSP + off]`, DWARF register 7. The unwinder
+path resolved bases with `_Unwind_GetGR(context, reg)`, and **`_Unwind_GetGR`
+is not a supported query for the stack-pointer column** — it returned garbage
+that the collector then wrote through. `_Unwind_GetCFA` is the supported way.
+
+SP-relative roots now derive their base from the CFA, and the arithmetic is
+**architecture-specific** rather than shared:
+
+* **x86-64** — `call` pushes the return address, so the CFA is the caller's
+  stack pointer before the call and the body stack pointer is
+  `CFA - 8 - stack_size`.
+* **aarch64** — `bl` writes the return address to `x30`; nothing is pushed, so
+  the body stack pointer is `CFA - stack_size`.
+
+Subtracting a return-address slot on aarch64 shifts every SP-relative root by a
+word. It would have stayed latent there because `chain_walkable` is true on
+aarch64, so the fast x29 walker runs and this path is only the fallback — the
+error would surface only once the fast walk bailed. The architecture's SP register number is a **runtime-local**
+constant, deliberately separate from the format's base tags — those stay
+aarch64-literal so the compiler's idea of the target and the runtime's
+`target_arch` can never disagree (see `gc_map.rs`).
+
+Measured on real x86-64 Linux hardware: **all ten gc-ratchet probes byte-match
+the pinned Node oracle** under `PERRY_RS4GC=1 PERRY_GC_FORCE_EVACUATE=1
+PERRY_GC_VERIFY_EVACUATION=1`, with `.perry_gcmap` present in every binary.
+
+The walker demonstrably ran rather than passing vacuously — root-source
+telemetry on x86-64 reports `walks=1, frames_visited=10, records_matched=1,
+locations_visited=2` on `02_survivor_promotion`, with `fp_walks=0` and
+`fallback_walks=1` (the fp-chain walker is aarch64-only, so the unwinder is the
+correct path there), and the evacuation moved objects
+(`retained_forwarded_stub_objects=5`).
+
+**The same telemetry on aarch64 has the same shape** (`7/0/1/1` vs `10/0/1/1`
+on the same probes, `2` locations on `09_try_catch_roots` on both), which is
+what makes this an equivalence result rather than a green light of unknown
+provenance.
+
+Caveat worth recording: those location counts are low **on both platforms**.
+The probes end in an explicit `gc()`, a manual collection from a shallow stack,
+so precise native roots are lightly exercised by this suite regardless of
+architecture. That is a pre-existing gate weakness, not something this change
+introduces, and it means "x86-64 matches aarch64" is a stronger claim here than
+"x86-64 is heavily exercised".