Update

[ghstack-poisoned]

Author: Gasoonjia

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-release/2.14
+ea89e4e90dc68302e8ef5cba3ddbdaa9d50d9512

.ci/docker/common/install_android.sh

@@ -79,10 +79,9 @@ install_sdk() {
   yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "build-tools;35.0.0"
   # And some more tools for future emulator tests
   yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "platform-tools"
-  # The 'tools' package (emulator) is not available on aarch64
-  if [ "$(uname -m)" != "aarch64" ]; then
-    yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "tools"
-  fi
+  # Google withdrew the 'tools' package, so installing it fails under set -e.
+  # cmdline-tools and platform-tools above replace it, and this image neither
+  # installs nor runs an emulator.
 }
 
 install_prerequiresites

.ci/scripts/tests/test_cuda_workflow.py

@@ -39,24 +39,17 @@ def test_build_matrix_preserves_existing_cuda_versions(self):
         )
         self.assertEqual(job["with"]["gpu-arch-version"], "${{ matrix.cuda-version }}")
 
-    def test_cuda134_driver_uses_matching_workflow_and_action_revision(self):
+    def test_cuda_builds_take_the_node_driver_on_an_unpinned_v3(self):
+        # This job used to pin linux_job_v2 to a test-infra SHA because that was
+        # the only ref carrying driver-version / driver-download-url, and it
+        # installed R615 for the 13.4 cell. On OSDC the driver belongs to the
+        # node and a pod cannot replace it, so the job takes what the node has.
         job = WORKFLOW["jobs"]["test-cuda-builds"]
-        workflow, revision = job["uses"].split("@")
         self.assertEqual(
-            workflow, "pytorch/test-infra/.github/workflows/linux_job_v2.yml"
-        )
-        self.assertRegex(revision, r"^[0-9a-f]{40}$")
-        self.assertEqual(job["with"]["test-infra-ref"], revision)
-        self.assertEqual(
-            job["with"]["driver-version"],
-            "${{ matrix.cuda-version == '13.4' && '615.71.09' || '580.65.06' }}",
-        )
-        self.assertEqual(
-            job["with"]["driver-download-url"],
-            "${{ matrix.cuda-version == '13.4' && "
-            "'https://download.nvidia.com/XFree86/Linux-x86_64/615.71.09/"
-            "NVIDIA-Linux-x86_64-615.71.09.run' || '' }}",
+            job["uses"], "pytorch/test-infra/.github/workflows/linux_job_v3.yml@main"
         )
+        for pin in ("test-infra-ref", "driver-version", "driver-download-url"):
+            self.assertNotIn(pin, job["with"])
 
     def test_cuda134_runtime_update_precedes_build_and_propagates_failure(self):
         script = WORKFLOW["jobs"]["test-cuda-builds"]["with"]["script"]

.ci/scripts/wheel/test_cpp_sdk.py

@@ -1098,7 +1098,8 @@ def test_shipped_headers_have_implementations(work_dir: Path) -> None:
             "#define ET_USE_THREADPOOL\n"
             "#include <executorch/runtime/kernel/thread_parallel_interface.h>\n"
             "using namespace executorch::extension;\n"
-            "int main() { return parallel_for(0, 1, 1, [](int64_t, int64_t) {}) ? 0 : 1; }\n"
+            "int main() { return get_thread_count() > 0 && "
+            "parallel_for(0, 1, 1, [](int64_t, int64_t) {}) ? 0 : 1; }\n"
         ),
     }
 

.github/workflows/_test_backend.yml

@@ -90,7 +90,13 @@ jobs:
   package-golden-artifacts:
     if: ${{ inputs.run-linux && inputs.save-goldens }}
     needs: test-backend-linux
-    runs-on: linux.2xlarge
+    # Repackaging artifacts needs no ExecuTorch toolchain, so this runs on a
+    # GitHub-hosted runner rather than moving to OSDC; the S3 upload that used
+    # to ride on the EC2 instance role assumes role/arc through OIDC instead.
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
     steps:
       - name: Download model test artifacts
         uses: actions/download-artifact@v4
@@ -124,6 +130,13 @@ jobs:
             echo "No golden artifacts found."
           fi
 
+      - name: Configure AWS credentials
+        if: ${{ hashFiles('golden_artifacts_*.zip') != '' }}
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/arc
+          aws-region: us-east-1
+
       - name: Upload golden artifacts to S3
         uses: seemethere/upload-artifact-s3@v5
         if: ${{ hashFiles('golden_artifacts_*.zip') != '' }}

.github/workflows/android-release-artifacts.yml

@@ -40,6 +40,10 @@ on:
     - cron: 0 10 * * *
 
 jobs:
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   check-if-aar-exists:
     name: check-if-aar-exists
     runs-on: ubuntu-22.04
@@ -71,16 +75,16 @@ jobs:
 
   build-aar:
     name: build-aar
-    needs: check-if-aar-exists
+    needs: [docker-image, check-if-aar-exists]
     if: ${{ !github.event.pull_request.head.repo.fork && needs.check-if-aar-exists.outputs.should-skip != 'true' }}
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
     secrets: inherit
     permissions:
       id-token: write
       contents: read
     with:
       secrets-env: EXECUTORCH_MAVEN_SIGNING_KEYID EXECUTORCH_MAVEN_SIGNING_PASSWORD EXECUTORCH_MAVEN_CENTRAL_PASSWORD EXECUTORCH_MAVEN_CENTRAL_USERNAME EXECUTORCH_MAVEN_SIGNING_GPG_KEY_CONTENTS
-      docker-image: ci-image:executorch-ubuntu-22.04-clang12-android
+      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-android-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: 'recursive'
       ref: ${{ github.sha }}
       timeout: 90

.github/workflows/apple.yml

@@ -126,7 +126,13 @@ jobs:
   # Upload the test demo app to S3
   upload-demo-ios:
     needs: build-demo-ios
-    runs-on: linux.2xlarge
+    # Moving an artifact from GitHub to S3 needs no ExecuTorch toolchain, so
+    # this runs on a GitHub-hosted runner rather than moving to OSDC; the upload
+    # that used to ride on the EC2 instance role assumes role/arc through OIDC.
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
     steps:
       - name: Download the artifacts from GitHub
         uses: actions/download-artifact@v4
@@ -142,6 +148,12 @@ jobs:
           set -eux
           ls -lah ./
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/arc
+          aws-region: us-east-1
+
       - name: Upload the artifacts to S3
         uses: seemethere/upload-artifact-s3@v5
         with:
@@ -163,8 +175,8 @@ jobs:
     uses: pytorch/test-infra/.github/workflows/mobile_job.yml@main
     with:
       device-type: ios
-      # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS
-      runner: linux.2xlarge
+      # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS.
+      runner: ubuntu-latest
       test-infra-ref: main
       # This is the ARN of ExecuTorch project on AWS
       project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:02a2cf0f-6d9b-45ee-ba1a-a086587469e6

.github/workflows/build-cadence-runner.yml

@@ -19,26 +19,31 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   # Same-repo PRs run on pull_request, which reads the PR's own workflow AND code
   # -- so CI changes, new test jobs, code, and tests are all validated pre-merge.
   # Fork PRs can't get credentials (OIDC) on pull_request, so Meta-exported forks
   # (labeled CLA Signed + meta-exported) run on pull_request_target instead. The
   # run condition is inlined per job (GitHub Actions has no YAML anchors and env
   # is unavailable in job-level if), so keep the copies in sync.
   cpu-build:
+    needs: docker-image
     if: >-
       github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
       (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository &&
       contains(github.event.pull_request.labels.*.name, 'CLA Signed') && contains(github.event.pull_request.labels.*.name, 'meta-exported'))
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
     permissions:
       id-token: write
       contents: read
     with:
       job-name: build
-      runner: linux.2xlarge
-      docker-image: ci-image:executorch-ubuntu-22.04-clang12
+      runner: mt-l-x86iavx512-8-64
+      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: recursive
       ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }}
       timeout: 90

.github/workflows/build-cmsis-pack.yml

@@ -44,15 +44,20 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   build-cmsis-pack:
+    needs: docker-image
     name: build-cmsis-pack
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
     permissions:
       id-token: write
       contents: read
     with:
-      runner: linux.2xlarge
-      docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk
+      runner: mt-l-x86iavx512-8-64
+      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 60

.github/workflows/build-presets.yml

@@ -13,6 +13,10 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   apple:
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
     strategy:
@@ -34,16 +38,20 @@ jobs:
         ${CONDA_RUN} cmake --build cmake-out -j$(( $(sysctl -n hw.ncpu) - 1 ))
 
   zephyr:
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    needs: docker-image
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
+    permissions:
+      id-token: write
+      contents: read
     strategy:
       fail-fast: false
       matrix:
         preset: [zephyr]
     with:
       job-name: build
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      runner: linux.2xlarge
-      docker-image: ci-image:executorch-ubuntu-22.04-zephyr-sdk
+      runner: mt-l-x86iavx512-8-64
+      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-zephyr-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: recursive
       timeout: 90
       script: |
@@ -73,26 +81,30 @@ jobs:
         cmake --preset ${{ matrix.preset }}
         cmake --build cmake-out -j$(( $(nproc) - 1 ))
   linux:
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    needs: docker-image
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
+    permissions:
+      id-token: write
+      contents: read
     strategy:
       fail-fast: false
       matrix:
         preset: [linux, pybind, llm]
-        runner: [linux.2xlarge, linux.arm64.2xlarge]
+        runner: [mt-l-x86iavx512-8-64, mt-l-arm64g4-16-62]
         docker-image: [executorch-ubuntu-22.04-clang12, executorch-ubuntu-22.04-gcc11-aarch64]
         # Excluding specific runner + docker image combinations that don't make sense:
-        #   - Excluding the ARM64 gcc image on the x86 runner (linux.2xlarge)
-        #   - Excluding the x86 clang image on the ARM64 runner (linux.arm64.2xlarge)
+        #   - Excluding the ARM64 gcc image on the x86 runner
+        #   - Excluding the x86 clang image on the ARM64 runner
         exclude:
-          - runner: linux.2xlarge
+          - runner: mt-l-x86iavx512-8-64
             docker-image: executorch-ubuntu-22.04-gcc11-aarch64
-          - runner: linux.arm64.2xlarge
+          - runner: mt-l-arm64g4-16-62
             docker-image: executorch-ubuntu-22.04-clang12
     with:
       job-name: build
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       runner: ${{ matrix.runner }}
-      docker-image: ci-image:${{ matrix.docker-image }}
+      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:${{ matrix.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: recursive
       timeout: 90
       script: |