benchmarks.yml

name: Performance Benchmarks

# Measures the workflow runtime's core latency metrics (TTFS, STSO, WO,
# fan-out — see packages/core/e2e/benchmark.test.ts for definitions) against a
# deployed workbench app and posts the results as a sticky PR comment. Re-runs update
# the same comment; previous results stay available in a collapsed history
# section (state is embedded in the comment body itself).
#
# Runs on pushes to main (against the production deployment) purely to produce
# baseline artifacts; PR runs download the most recent main baseline and show
# avg-latency deltas against it in the comment.

on:
  pull_request:
    branches: [main]
  push:
    branches: [main]
  workflow_dispatch: # Allow manual triggers

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  ci-scope:
    name: Detect CI Scope
    runs-on: ubuntu-latest
    outputs:
      fast-path: ${{ steps.scope.outputs.runtime-fast-path }}
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4

      - name: Classify changed files
        id: scope
        uses: ./.github/actions/detect-ci-scope
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}

  # Phase 0: Update the PR comment to show benchmarks are running. Previous
  # results (and the collapsed history) are preserved from the existing
  # comment body.
  pr-comment-start:
    name: Create PR Comment
    runs-on: ubuntu-latest
    needs: ci-scope
    if: >-
      github.event_name == 'pull_request' &&
      !startsWith(github.head_ref, 'changeset-release/') &&
      !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') &&
      needs.ci-scope.outputs.fast-path != 'true'
    timeout-minutes: 5

    steps:
      - uses: actions/checkout@v4

      - name: Find existing benchmark comment
        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
        id: find-comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: '<!-- benchmark-results -->'

      - name: Save existing comment body
        if: steps.find-comment.outputs.comment-id != ''
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const comment = await github.rest.issues.getComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: ${{ steps.find-comment.outputs.comment-id }}
            });
            fs.writeFileSync(`${process.env.RUNNER_TEMP}/previous-comment.md`, comment.data.body ?? '');

      - name: Render running comment
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          node .github/scripts/render-benchmark-comment.mjs \
            --status running \
            --previous-body "$RUNNER_TEMP/previous-comment.md" \
            --commit "$HEAD_SHA" \
            --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            --output "$RUNNER_TEMP/comment.md"

      - name: Update PR comment
        uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
        with:
          header: benchmark-results
          path: ${{ runner.temp }}/comment.md

  # Phase 1: Run the benchmarks. Currently Vercel-only; to add another
  # backend (e.g. postgres or local), add a matrix entry with
  # `world: postgres` / `world: local` and gate the Vercel-specific steps —
  # the runner (packages/core/e2e/benchmark.test.ts) already selects its
  # backend from the same env vars as the e2e tests. Streaming delivery
  # performance is measured in durabench; stream correctness stays in E2E.
  benchmark:
    name: Benchmark (${{ matrix.target.world }}, ${{ matrix.target.app }})
    runs-on: ubuntu-latest
    needs: ci-scope
    # `changeset-release/*` PRs are not deployed (every project's vercel.json
    # sets `git.deploymentEnabled` false for that branch, because the branch is
    # force-pushed to main's HEAD SHA and a same-SHA preview deployment
    # clobbers the per-SHA Vercel commit status), so there is nothing to wait
    # for below. Benchmarking them would also be meaningless: the code is
    # main's, so it would only compare main's baseline against itself.
    if: >-
      !startsWith(github.head_ref, 'changeset-release/') &&
      !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') &&
      (needs.ci-scope.outputs.fast-path != 'true' ||
      github.event_name == 'workflow_dispatch')
    timeout-minutes: 60
    permissions:
      id-token: write
      contents: read
    env:
      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
    strategy:
      fail-fast: false
      matrix:
        target:
          - world: vercel
            app: nextjs-turbopack
            project-id: "prj_yjkM7UdHliv8bfxZ1sMJQf1pMpdi"
            project-slug: "example-nextjs-workflow-turbopack"

    steps:
      - uses: actions/checkout@v4

      - name: Setup environment
        uses: ./.github/actions/setup-workflow-dev
        with:
          build-packages: 'false'

      # Builds the packages the benchmark runner imports (same pattern as the
      # e2e-vercel jobs in tests.yml); hits the turbo remote cache in CI.
      - name: Build packages
        run: pnpm turbo run build --filter='@workflow/cli'

      - name: Resolve Vercel deployment
        id: deployment
        if: matrix.target.world == 'vercel'
        uses: ./.github/actions/resolve-vercel-deployment
        with:
          project-id: ${{ matrix.target.project-id }}
          token: ${{ secrets.VERCEL_LABS_TOKEN }}

      - name: Run benchmarks
        env:
          DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }}
          APP_NAME: ${{ matrix.target.app }}
          VERCEL_DEPLOYMENT_ID: ${{ steps.deployment.outputs.deployment-id }}
          WORKFLOW_VERCEL_ENV: ${{ steps.deployment.outputs.environment }}
          VERCEL_WORKFLOW_SERVER_URL: ${{ steps.deployment.outputs.environment == 'preview' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }}
          WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }}
          WORKFLOW_VERCEL_TEAM: "team_nO2mCG4W8IxPIeKoSsqwAxxB"
          WORKFLOW_VERCEL_PROJECT: ${{ matrix.target.project-id }}
          WORKFLOW_VERCEL_PROJECT_SLUG: ${{ matrix.target.project-slug }}
          # Report the PR head SHA (not the synthetic merge commit) in results
          GITHUB_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          # Trusted-sources OIDC token is minted on demand by
          # scripts/trusted-sources-headers.mjs via the runner's
          # ACTIONS_ID_TOKEN_REQUEST_URL/_TOKEN env vars (auto-set when
          # `permissions: id-token: write` is on the job).
          # See: https://vercel.com/docs/deployment-protection/methods-to-bypass-deployment-protection/trusted-sources
        run: |
          pnpm vitest run packages/core/e2e/benchmark.test.ts

      - name: Upload benchmark results
        # Partial results are still useful when a scenario failed
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: bench-results-${{ matrix.target.app }}-${{ matrix.target.world }}
          path: bench-results-*.json
          if-no-files-found: ignore

  # Phase 3: Update the PR comment with the results (collapsing history)
  comment:
    name: Benchmark Comment
    runs-on: ubuntu-latest
    needs: [ci-scope, benchmark]
    if: >-
      always() && !cancelled() &&
      github.event_name == 'pull_request' &&
      !startsWith(github.head_ref, 'changeset-release/') &&
      !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') &&
      needs.ci-scope.outputs.fast-path != 'true'
    timeout-minutes: 10

    steps:
      - uses: actions/checkout@v4

      - name: Download benchmark results
        continue-on-error: true
        uses: actions/download-artifact@v4
        with:
          pattern: bench-results-*
          path: benchmark-results
          merge-multiple: true

      # Baseline for the avg-delta column: results from the most recent
      # successful run of this workflow on main. Uses gh (first-party) rather
      # than a third-party cross-run artifact action. Filtering to push /
      # workflow_dispatch events matters for more than tidiness: it excludes
      # artifacts uploaded by fork PR runs, whose head branch can also be
      # named "main" (artifact poisoning).
      - name: Download baseline results from main
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          run_id=$(gh run list \
            --repo "$GITHUB_REPOSITORY" \
            --workflow benchmarks.yml \
            --branch main \
            --status success \
            --limit 20 \
            --json databaseId,event \
            --jq '[.[] | select(.event == "push" or .event == "workflow_dispatch")][0].databaseId // empty')
          if [ -z "$run_id" ]; then
            echo "No successful benchmark run found on main; skipping baseline"
            exit 0
          fi
          echo "Using baseline artifacts from run $run_id"
          gh run download "$run_id" \
            --repo "$GITHUB_REPOSITORY" \
            --pattern 'bench-results-*' \
            --dir baseline-results \
            || echo "Run $run_id has no benchmark artifacts; skipping baseline"

      - name: Find existing benchmark comment
        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
        id: find-comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: '<!-- benchmark-results -->'

      - name: Save existing comment body
        if: steps.find-comment.outputs.comment-id != ''
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const comment = await github.rest.issues.getComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: ${{ steps.find-comment.outputs.comment-id }}
            });
            fs.writeFileSync(`${process.env.RUNNER_TEMP}/previous-comment.md`, comment.data.body ?? '');

      - name: Render results comment
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          BENCH_STATUS: ${{ needs.benchmark.result == 'success' && 'completed' || 'failed' }}
        run: |
          node .github/scripts/render-benchmark-comment.mjs \
            --status "$BENCH_STATUS" \
            --results-dir benchmark-results \
            --baseline-dir baseline-results \
            --previous-body "$RUNNER_TEMP/previous-comment.md" \
            --commit "$HEAD_SHA" \
            --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            --output "$RUNNER_TEMP/comment.md"
          cat "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY"

      - name: Update PR comment
        uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
        with:
          header: benchmark-results
          path: ${{ runner.temp }}/comment.md