Repository navigation
Version Packages #16202
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Benchmarks | |
| # Measures the workflow runtime's core latency metrics (TTFS, STSO, WO, | |
| # fan-out — see packages/core/e2e/benchmark.test.ts for definitions) against a | |
| # deployed workbench app and posts the results as a sticky PR comment. Re-runs update | |
| # the same comment; previous results stay available in a collapsed history | |
| # section (state is embedded in the comment body itself). | |
| # | |
| # Runs on pushes to main (against the production deployment) purely to produce | |
| # baseline artifacts; PR runs download the most recent main baseline and show | |
| # avg-latency deltas against it in the comment. | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| workflow_dispatch: # Allow manual triggers | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| ci-scope: | |
| name: Detect CI Scope | |
| runs-on: ubuntu-latest | |
| outputs: | |
| fast-path: ${{ steps.scope.outputs.runtime-fast-path }} | |
| steps: | |
| - name: Checkout Repo | |
| uses: actions/checkout@v4 | |
| - name: Classify changed files | |
| id: scope | |
| uses: ./.github/actions/detect-ci-scope | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Phase 0: Update the PR comment to show benchmarks are running. Previous | |
| # results (and the collapsed history) are preserved from the existing | |
| # comment body. | |
| pr-comment-start: | |
| name: Create PR Comment | |
| runs-on: ubuntu-latest | |
| needs: ci-scope | |
| if: >- | |
| github.event_name == 'pull_request' && | |
| !startsWith(github.head_ref, 'changeset-release/') && | |
| !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') && | |
| needs.ci-scope.outputs.fast-path != 'true' | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Find existing benchmark comment | |
| uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0 | |
| id: find-comment | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-author: 'github-actions[bot]' | |
| body-includes: '<!-- benchmark-results -->' | |
| - name: Save existing comment body | |
| if: steps.find-comment.outputs.comment-id != '' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const comment = await github.rest.issues.getComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: ${{ steps.find-comment.outputs.comment-id }} | |
| }); | |
| fs.writeFileSync(`${process.env.RUNNER_TEMP}/previous-comment.md`, comment.data.body ?? ''); | |
| - name: Render running comment | |
| env: | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| node .github/scripts/render-benchmark-comment.mjs \ | |
| --status running \ | |
| --previous-body "$RUNNER_TEMP/previous-comment.md" \ | |
| --commit "$HEAD_SHA" \ | |
| --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ | |
| --output "$RUNNER_TEMP/comment.md" | |
| - name: Update PR comment | |
| uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 | |
| with: | |
| header: benchmark-results | |
| path: ${{ runner.temp }}/comment.md | |
| # Phase 1: Run the benchmarks. Currently Vercel-only; to add another | |
| # backend (e.g. postgres or local), add a matrix entry with | |
| # `world: postgres` / `world: local` and gate the Vercel-specific steps — | |
| # the runner (packages/core/e2e/benchmark.test.ts) already selects its | |
| # backend from the same env vars as the e2e tests. Streaming delivery | |
| # performance is measured in durabench; stream correctness stays in E2E. | |
| benchmark: | |
| name: Benchmark (${{ matrix.target.world }}, ${{ matrix.target.app }}) | |
| runs-on: ubuntu-latest | |
| needs: ci-scope | |
| # `changeset-release/*` PRs are not deployed (every project's vercel.json | |
| # sets `git.deploymentEnabled` false for that branch, because the branch is | |
| # force-pushed to main's HEAD SHA and a same-SHA preview deployment | |
| # clobbers the per-SHA Vercel commit status), so there is nothing to wait | |
| # for below. Benchmarking them would also be meaningless: the code is | |
| # main's, so it would only compare main's baseline against itself. | |
| if: >- | |
| !startsWith(github.head_ref, 'changeset-release/') && | |
| !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') && | |
| (needs.ci-scope.outputs.fast-path != 'true' || | |
| github.event_name == 'workflow_dispatch') | |
| timeout-minutes: 60 | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: ${{ vars.TURBO_TEAM }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: | |
| - world: vercel | |
| app: nextjs-turbopack | |
| project-id: "prj_yjkM7UdHliv8bfxZ1sMJQf1pMpdi" | |
| project-slug: "example-nextjs-workflow-turbopack" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-workflow-dev | |
| with: | |
| build-packages: 'false' | |
| # Builds the packages the benchmark runner imports (same pattern as the | |
| # e2e-vercel jobs in tests.yml); hits the turbo remote cache in CI. | |
| - name: Build packages | |
| run: pnpm turbo run build --filter='@workflow/cli' | |
| - name: Resolve Vercel deployment | |
| id: deployment | |
| if: matrix.target.world == 'vercel' | |
| uses: ./.github/actions/resolve-vercel-deployment | |
| with: | |
| project-id: ${{ matrix.target.project-id }} | |
| token: ${{ secrets.VERCEL_LABS_TOKEN }} | |
| - name: Run benchmarks | |
| env: | |
| DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }} | |
| APP_NAME: ${{ matrix.target.app }} | |
| VERCEL_DEPLOYMENT_ID: ${{ steps.deployment.outputs.deployment-id }} | |
| WORKFLOW_VERCEL_ENV: ${{ steps.deployment.outputs.environment }} | |
| VERCEL_WORKFLOW_SERVER_URL: ${{ steps.deployment.outputs.environment == 'preview' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} | |
| WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }} | |
| WORKFLOW_VERCEL_TEAM: "team_nO2mCG4W8IxPIeKoSsqwAxxB" | |
| WORKFLOW_VERCEL_PROJECT: ${{ matrix.target.project-id }} | |
| WORKFLOW_VERCEL_PROJECT_SLUG: ${{ matrix.target.project-slug }} | |
| # Report the PR head SHA (not the synthetic merge commit) in results | |
| GITHUB_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| # Trusted-sources OIDC token is minted on demand by | |
| # scripts/trusted-sources-headers.mjs via the runner's | |
| # ACTIONS_ID_TOKEN_REQUEST_URL/_TOKEN env vars (auto-set when | |
| # `permissions: id-token: write` is on the job). | |
| # See: https://vercel.com/docs/deployment-protection/methods-to-bypass-deployment-protection/trusted-sources | |
| run: | | |
| pnpm vitest run packages/core/e2e/benchmark.test.ts | |
| - name: Upload benchmark results | |
| # Partial results are still useful when a scenario failed | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bench-results-${{ matrix.target.app }}-${{ matrix.target.world }} | |
| path: bench-results-*.json | |
| if-no-files-found: ignore | |
| # Phase 3: Update the PR comment with the results (collapsing history) | |
| comment: | |
| name: Benchmark Comment | |
| runs-on: ubuntu-latest | |
| needs: [ci-scope, benchmark] | |
| if: >- | |
| always() && !cancelled() && | |
| github.event_name == 'pull_request' && | |
| !startsWith(github.head_ref, 'changeset-release/') && | |
| !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') && | |
| needs.ci-scope.outputs.fast-path != 'true' | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download benchmark results | |
| continue-on-error: true | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: bench-results-* | |
| path: benchmark-results | |
| merge-multiple: true | |
| # Baseline for the avg-delta column: results from the most recent | |
| # successful run of this workflow on main. Uses gh (first-party) rather | |
| # than a third-party cross-run artifact action. Filtering to push / | |
| # workflow_dispatch events matters for more than tidiness: it excludes | |
| # artifacts uploaded by fork PR runs, whose head branch can also be | |
| # named "main" (artifact poisoning). | |
| - name: Download baseline results from main | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| run_id=$(gh run list \ | |
| --repo "$GITHUB_REPOSITORY" \ | |
| --workflow benchmarks.yml \ | |
| --branch main \ | |
| --status success \ | |
| --limit 20 \ | |
| --json databaseId,event \ | |
| --jq '[.[] | select(.event == "push" or .event == "workflow_dispatch")][0].databaseId // empty') | |
| if [ -z "$run_id" ]; then | |
| echo "No successful benchmark run found on main; skipping baseline" | |
| exit 0 | |
| fi | |
| echo "Using baseline artifacts from run $run_id" | |
| gh run download "$run_id" \ | |
| --repo "$GITHUB_REPOSITORY" \ | |
| --pattern 'bench-results-*' \ | |
| --dir baseline-results \ | |
| || echo "Run $run_id has no benchmark artifacts; skipping baseline" | |
| - name: Find existing benchmark comment | |
| uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0 | |
| id: find-comment | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-author: 'github-actions[bot]' | |
| body-includes: '<!-- benchmark-results -->' | |
| - name: Save existing comment body | |
| if: steps.find-comment.outputs.comment-id != '' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const comment = await github.rest.issues.getComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: ${{ steps.find-comment.outputs.comment-id }} | |
| }); | |
| fs.writeFileSync(`${process.env.RUNNER_TEMP}/previous-comment.md`, comment.data.body ?? ''); | |
| - name: Render results comment | |
| env: | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| BENCH_STATUS: ${{ needs.benchmark.result == 'success' && 'completed' || 'failed' }} | |
| run: | | |
| node .github/scripts/render-benchmark-comment.mjs \ | |
| --status "$BENCH_STATUS" \ | |
| --results-dir benchmark-results \ | |
| --baseline-dir baseline-results \ | |
| --previous-body "$RUNNER_TEMP/previous-comment.md" \ | |
| --commit "$HEAD_SHA" \ | |
| --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ | |
| --output "$RUNNER_TEMP/comment.md" | |
| cat "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Update PR comment | |
| uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 | |
| with: | |
| header: benchmark-results | |
| path: ${{ runner.temp }}/comment.md |