diff --git a/.claude/skills/playwright-test-results/SKILL.md b/.claude/skills/playwright-test-results/SKILL.md new file mode 100644 index 0000000000000..aace3aff88627 --- /dev/null +++ b/.claude/skills/playwright-test-results/SKILL.md @@ -0,0 +1,135 @@ +--- +name: playwright-test-results +description: Query Playwright CI test results from the aggregated DuckDB database. Answers questions about flaky tests, failure rates, slow tests, and per-run/SHA/PR results without hunting through GitHub artifacts. +user_invocable: true +--- + +# Playwright Test Results (DuckDB) + +A single DuckDB file holds recent Playwright CI test results, so you can answer +questions about failures, flakiness, and slow tests with plain SQL. It is +refreshed every few hours. + +## Get the database + +Download the latest snapshot: + +```bash +npm ci # first time only, from the repo root +GITHUB_TOKEN=$(gh auth token) node utils/test-results-db/cli.ts download +``` + +The snapshot may be missing the newest runs. To top it up locally, run `update`: + +```bash +GITHUB_TOKEN=$(gh auth token) node utils/test-results-db/cli.ts update --lookback-days 3 +``` + +Query it with the `duckdb` CLI (or any DuckDB client): + +```bash +duckdb utils/test-results-db/test-results.duckdb "SELECT count(*) FROM test_results" +``` + +## Schema + +Single table `test_results`, one row per test result (**one row per retry**). +The columns are inferred from the parquet the reporter emits +(`tests/config/parquetReporter.ts`), plus two trailing columns this CLI adds: + +| Column | Meaning | +| --- | --- | +| `run_id`, `run_attempt` | GitHub Actions run identity | +| `run_started_at` | when the run started | +| `workflow_name` | e.g. `tests 1` / `tests 2` / `tests others` / `MCP` | +| `event` | `push` / `pull_request` | +| `head_sha`, `head_branch`, `pr_number` | what was tested | +| `bot_name` | e.g. `chromium-ubuntu-22.04-node20`, `webkit-macos-15-large` — the CI bot. **OS and arch are encoded here**; there is no separate os column. | +| `project_name` | CI project = browser + suite, e.g. `chromium-page`, `webkit-library`, `playwright-test` | +| `test_title` | title path within the file, joined by ` › ` (`describe › test`) | +| `file`, `line`, `column_number` | source location (file is relative to repo root) | +| `expected_status` | `passed` / `skipped` / ... | +| `status` | actual result: `passed` / `failed` / `timedOut` / `skipped` / `interrupted` | +| `retry` | 0 = first attempt | +| `result_started_at` | when this attempt started | +| `duration_ms` | result duration | +| `error_message` | all errors joined, ANSI-stripped (NULL when none) | +| `tags` | **list** of strings, e.g. `['@slow', '@flaky']` (use list functions / `list_contains`) | +| `annotations` | list of `{type, description}` structs, e.g. `[{'type': 'skip', 'description': 'flaky on CI'}]` (empty list when none) | +| `artifact_id` | the GitHub artifact this row came from (dedupe key) | +| `ingested_at` | debug only — when this row was imported | + +Notes: +- **A test is identified by `(project_name, file, test_title)`** — group on that + tuple. (Playwright's `test_id` hash is deliberately not stored; those three + columns are its pre-image.) +- **Flakiness is derived**, not stored. The signal that matters most is + **cross-run**: a test whose *final* verdict (after retries) flips between + runs — green in some, red in others. A separate **within-run** flake is a + test a retry rescued inside a single run (`failed`→`passed`). +- **Real failures vs intentional ones:** filter `expected_status = 'passed'`. + Tests marked `test.fail()` record `status='failed'` *with* + `expected_status='failed'` and would otherwise dominate any "most failing" list. +- The db is size-capped by **run count**: the oldest whole runs are evicted over + time, so it holds a recent window, not full history. + +## Example queries + +Group tests by `(project_name, file, test_title)` and (for failure/flakiness) +scope to `expected_status = 'passed'` so intentional `test.fail()` tests don't +skew the results. + +**Flaky across runs** — the test's final verdict flips between runs (this is +what makes a red CI run ambiguous). `least(failed_runs, passed_runs)` ranks +genuinely bimodal tests above both always-broken and one-off failures: + +```sql +WITH per_run AS ( + SELECT project_name, file, test_title, run_id, run_attempt, + arg_max(status, retry) AS final_status, + any_value(expected_status) AS expected + FROM test_results + GROUP BY project_name, file, test_title, run_id, run_attempt) +SELECT project_name, test_title, + count(*) AS runs, + count(*) FILTER (WHERE final_status IN ('failed','timedOut')) AS failed_runs, + count(*) FILTER (WHERE final_status = 'passed') AS passed_runs, + round(100.0 * count(*) FILTER (WHERE final_status IN ('failed','timedOut')) + / count(*), 1) AS fail_pct +FROM per_run +WHERE expected = 'passed' +GROUP BY project_name, test_title +HAVING failed_runs > 0 AND passed_runs > 0 AND runs >= 10 +ORDER BY least(failed_runs, passed_runs) DESC, failed_runs DESC +LIMIT 20; +``` + +**Filter by tag** (`tags` is a list, not a string): + +```sql +SELECT project_name, test_title, count(*) AS runs +FROM test_results +WHERE list_contains(tags, '@slow') +GROUP BY project_name, test_title +ORDER BY runs DESC +LIMIT 20; +``` + +## Fetching the full detail + +The db stores per-result summaries. For the full step tree / attachments / stdio, +fetch the original blob report for that run, if the run uploaded one. A row +identifies it by `run_id` + `bot_name`: the run's blob artifact is named +`blob-report-`. + +```bash +# List the run's blob artifacts and find the one for this bot_name: +gh api /repos/microsoft/playwright/actions/runs//artifacts \ + --jq '.artifacts[] | select(.name | startswith("blob-report")) | {id, name}' + +# Download it (name == "blob-report-"): +gh api /repos/microsoft/playwright/actions/artifacts//zip > blob.zip +``` + +Blob and parquet artifacts have a 7-day retention, so this works only for recent +runs; the db itself retains summaries longer (until run-count eviction). diff --git a/.github/workflows/update_test_results_db.yml b/.github/workflows/update_test_results_db.yml new file mode 100644 index 0000000000000..96fecd024437b --- /dev/null +++ b/.github/workflows/update_test_results_db.yml @@ -0,0 +1,46 @@ +name: "Update test results DB" +on: + workflow_dispatch: + schedule: + - cron: "0 */3 * * *" + +concurrency: + group: test-results-db + cancel-in-progress: false + +jobs: + update: + name: Update DuckDB + runs-on: ubuntu-22.04 + timeout-minutes: 60 + if: github.repository == 'microsoft/playwright' + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: 26 + - run: npm ci + - name: Download current database + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: node utils/test-results-db/cli.ts download + - name: Ingest new test results + id: ingest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NODE_OPTIONS: --max-old-space-size=8192 + run: node utils/test-results-db/cli.ts update --lookback-days 7 --concurrency 32 + - name: Truncate to run cap + if: steps.ingest.outputs.imported != '0' + run: node utils/test-results-db/cli.ts truncate --max-runs 2000 + - name: Upload database + if: steps.ingest.outputs.imported != '0' + uses: actions/upload-artifact@v7 + with: + name: test-results-db + path: utils/test-results-db/test-results.duckdb + retention-days: 7 + overwrite: true diff --git a/utils/test-results-db/.gitignore b/utils/test-results-db/.gitignore new file mode 100644 index 0000000000000..2570c551208a7 --- /dev/null +++ b/utils/test-results-db/.gitignore @@ -0,0 +1,3 @@ +# The maintained database and DuckDB sidecar files (WAL, compaction temp). +*.duckdb +*.duckdb.* diff --git a/utils/test-results-db/cli.ts b/utils/test-results-db/cli.ts new file mode 100644 index 0000000000000..d06890cab4e4c --- /dev/null +++ b/utils/test-results-db/cli.ts @@ -0,0 +1,121 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import { fileURLToPath } from 'url'; +import { parseArgs } from 'util'; + +import { cmdDownload } from './download.ts'; +import { cmdTruncate } from './truncate.ts'; +import { cmdUpdate } from './update.ts'; + +const USAGE = `Usage: node utils/test-results-db/cli.ts [options] + +Compacts the per-run parquet CI artifacts into a single queryable DuckDB file. + +Commands: + download Fetch the latest maintained database artifact. + Starts a fresh database if none exists yet. + update [options] Ingest parquet artifacts that aren't in the database yet. + --lookback-days How many days back to scan (default 7). + --concurrency Parallel downloads per batch (default 16). + --stop-after-seen Stop after this many consecutive already-ingested + artifacts (default 100). The list is newest-first, so + this short-circuits the scan once caught up. + truncate --max-runs Keep only the newest runs, delete the rest, compact. + +Environment: + GITHUB_TOKEN Required for 'download' and 'update'. + TRDB_DB_PATH Database file path (default utils/test-results-db/test-results.duckdb). +`; + +function defaultDbPath(): string { + const here = path.dirname(fileURLToPath(import.meta.url)); + return path.join(here, 'test-results.duckdb'); +} + +const UPDATE_OPTIONS = { + 'lookback-days': { type: 'string' }, + 'concurrency': { type: 'string' }, + 'stop-after-seen': { type: 'string' }, +} as const; + +const TRUNCATE_OPTIONS = { + 'max-runs': { type: 'string' }, +} as const; + +type Flags = Record; + +function intFlag(flags: Flags, name: string, fallback: number): number { + const raw = flags[name]; + if (raw === undefined) + return fallback; + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) + throw new Error(`--${name} must be a positive integer, got "${raw}"`); + return value; +} + +function requireToken(): string { + const token = process.env.GITHUB_TOKEN; + if (!token) + throw new Error('GITHUB_TOKEN is required for this command.'); + return token; +} + +async function main(): Promise { + const [command, ...rest] = process.argv.slice(2); + const dbPath = process.env.TRDB_DB_PATH || defaultDbPath(); + + switch (command) { + case 'download': { + await cmdDownload(dbPath, requireToken()); + break; + } + case 'update': { + const { values } = parseArgs({ args: rest, options: UPDATE_OPTIONS, allowPositionals: false }); + await cmdUpdate(dbPath, requireToken(), { + lookbackDays: intFlag(values, 'lookback-days', 7), + concurrency: intFlag(values, 'concurrency', 16), + stopAfterSeen: intFlag(values, 'stop-after-seen', 100), + }); + break; + } + case 'truncate': { + const { values } = parseArgs({ args: rest, options: TRUNCATE_OPTIONS, allowPositionals: false }); + if (values['max-runs'] === undefined) + throw new Error('truncate requires --max-runs '); + await cmdTruncate(dbPath, intFlag(values, 'max-runs', 0)); + break; + } + case undefined: + case 'help': + case '--help': + case '-h': { + process.stdout.write(USAGE); + break; + } + default: { + process.stderr.write(`Unknown command: ${command}\n\n${USAGE}`); + process.exitCode = 1; + } + } +} + +main().catch(error => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); diff --git a/utils/test-results-db/db.ts b/utils/test-results-db/db.ts new file mode 100644 index 0000000000000..0aabfa81315b3 --- /dev/null +++ b/utils/test-results-db/db.ts @@ -0,0 +1,156 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; + +import { DuckDBInstance } from '@duckdb/node-api'; + +import type { DuckDBConnection } from '@duckdb/node-api'; + +const TABLE_NAME = 'test_results'; + +// The parquet already carries the full per-result schema (see +// tests/config/parquetReporter.ts). We deliberately do not re-declare those +// columns here: the table is created lazily from the first parquet we ingest, +// so it always matches whatever the reporter currently emits, plus these two +// trailing columns we add ourselves: +// - artifact_id: the GitHub artifact this row came from (our dedupe key) +// - ingested_at: when we imported it (debug only) +const INGEST_SELECT = + `SELECT *, $id AS artifact_id, now() AS ingested_at FROM read_parquet($file)`; + +// A row lives in the `test_results` table and originates from a GitHub artifact +// identified by an opaque string id. +export class TestResultsDb { + private _instance: DuckDBInstance; + private _conn: DuckDBConnection; + readonly path: string; + + private constructor(instance: DuckDBInstance, conn: DuckDBConnection, path: string) { + this._instance = instance; + this._conn = conn; + this.path = path; + } + + static async open(path: string): Promise { + const instance = await DuckDBInstance.create(path); + const conn = await instance.connect(); + return new TestResultsDb(instance, conn, path); + } + + private async _tableExists(): Promise { + const reader = await this._conn.runAndReadAll( + `SELECT 1 FROM information_schema.tables WHERE table_name = $name`, + { name: TABLE_NAME }); + return reader.getRows().length > 0; + } + + // The set of artifact ids already imported. Empty on a fresh (table-less) db. + async ingestedArtifactIds(): Promise> { + const ids = new Set(); + if (!await this._tableExists()) + return ids; + const reader = await this._conn.runAndReadAll( + `SELECT DISTINCT artifact_id FROM ${TABLE_NAME}`); + for (const row of reader.getRows()) + ids.add(String(row[0])); + return ids; + } + + // Ingest one parquet file, tagging every row with `artifactId`. The table is + // created (schema inferred) on the first ingest; later ingests are matched by + // column name, so a reordered/extended parquet schema still lands correctly. + async ingestParquet(parquetFile: string, artifactId: string): Promise { + const params = { id: artifactId, file: parquetFile }; + if (!await this._tableExists()) + await this._conn.run(`CREATE TABLE ${TABLE_NAME} AS ${INGEST_SELECT} LIMIT 0`, params); + await this._conn.run(`INSERT INTO ${TABLE_NAME} BY NAME ${INGEST_SELECT}`, params); + } + + async runCount(): Promise { + if (!await this._tableExists()) + return 0; + const reader = await this._conn.runAndReadAll( + `SELECT count(DISTINCT (run_id, run_attempt)) FROM ${TABLE_NAME}`); + return Number(reader.getRows()[0][0]); + } + + async rowCount(): Promise { + if (!await this._tableExists()) + return 0; + const reader = await this._conn.runAndReadAll(`SELECT count(*) FROM ${TABLE_NAME}`); + return Number(reader.getRows()[0][0]); + } + + // Keep the newest `maxRuns` runs (a run is a (run_id, run_attempt) pair, + // ordered by run_started_at), delete the rest, then compact to reclaim disk. + async truncateToRuns(maxRuns: number): Promise { + if (!await this._tableExists()) + return; + await this._conn.run( + `DELETE FROM ${TABLE_NAME} + WHERE (run_id, run_attempt) NOT IN ( + SELECT run_id, run_attempt FROM ${TABLE_NAME} + GROUP BY run_id, run_attempt + ORDER BY max(run_started_at) DESC NULLS LAST + LIMIT $n)`, + { n: maxRuns }); + await this._compact(); + } + + // Copy the live rows into a fresh database file and swap it in — DuckDB's + // DELETE leaves the space allocated, so this is what actually shrinks the file. + private async _compact(): Promise { + const tmpPath = `${this.path}.compact.tmp`; + for (const p of [tmpPath, `${tmpPath}.wal`]) { + if (fs.existsSync(p)) + fs.rmSync(p); + } + await this._conn.run(`ATTACH '${tmpPath.replace(/'/g, `''`)}' AS compacted`); + await this._conn.run(`CREATE TABLE compacted.${TABLE_NAME} AS SELECT * FROM ${TABLE_NAME}`); + await this._conn.run(`DETACH compacted`); + this.close(); + + fs.rmSync(this.path); + if (fs.existsSync(`${this.path}.wal`)) + fs.rmSync(`${this.path}.wal`); + fs.renameSync(tmpPath, this.path); + + this._instance = await DuckDBInstance.create(this.path); + this._conn = await this._instance.connect(); + } + + close(): void { + this._conn.closeSync(); + this._instance.closeSync(); + } +} + +export function fileSize(path: string): number { + try { + return fs.statSync(path).size; + } catch { + return 0; + } +} + +export function formatBytes(bytes: number): string { + if (bytes < 1024) + return `${bytes} B`; + if (bytes < 1024 * 1024) + return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} diff --git a/utils/test-results-db/download.ts b/utils/test-results-db/download.ts new file mode 100644 index 0000000000000..f5abdb72552fa --- /dev/null +++ b/utils/test-results-db/download.ts @@ -0,0 +1,37 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TestResultsDb, fileSize, formatBytes } from './db.ts'; +import { GitHubClient, extractSingle } from './github.ts'; + +const DB_ARTIFACT_NAME = 'test-results-db'; + +// Fetch the latest maintained database artifact. If none exists yet (the cron +// hasn't produced one), start from a fresh empty file. +export async function cmdDownload(dbPath: string, token: string): Promise { + const github = new GitHubClient(token); + const artifactId = await github.findLatestArtifact(DB_ARTIFACT_NAME); + if (!artifactId) { + const db = await TestResultsDb.open(dbPath); + db.close(); + console.log(`No "${DB_ARTIFACT_NAME}" artifact yet; started a fresh database at ${dbPath}`); + return; + } + console.log(`Downloading "${DB_ARTIFACT_NAME}" artifact #${artifactId} ...`); + const zip = await github.downloadArtifactZip(artifactId); + await extractSingle(zip, '.duckdb', dbPath); + console.log(`Downloaded database to ${dbPath} (${formatBytes(fileSize(dbPath))})`); +} diff --git a/utils/test-results-db/github.ts b/utils/test-results-db/github.ts new file mode 100644 index 0000000000000..97564e83422d7 --- /dev/null +++ b/utils/test-results-db/github.ts @@ -0,0 +1,186 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; + +import yauzl from 'yauzl'; + +export type Artifact = { + id: string; + name: string; +}; + +type ListOptions = { + ingested: Set; + lookbackDays: number; + stopAfterSeen: number; +}; + +type RawArtifact = { + id: number; + name: string; + expired: boolean; + created_at: string; +}; + +// Thin GitHub REST client over global fetch. Only the three endpoints this CLI +// needs: list artifacts, list by name, and download an artifact zip. +export class GitHubClient { + private _base: string; + private _headers: Record; + + constructor(token: string, repo: string = 'microsoft/playwright') { + if (!token) + throw new Error('A GitHub token is required (set GITHUB_TOKEN).'); + this._base = `https://api.github.com/repos/${repo}`; + this._headers = { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + }; + } + + // Return the not-yet-ingested artifacts matching `prefix`, walking the list + // from the top and stopping early once we're safely into the already-ingested + // region. + // + // The list is ordered by descending artifact id, and GitHub assigns ids as a + // monotonic creation-order sequence (verified: no inversions across a 1000- + // artifact sample). So the newest artifacts are always at the head. An artifact + // gets its id when its upload *starts* but only appears in the list once the + // upload *finalizes*, so the single way one can surface below where a prior + // scan stopped is a still-in-flight upload finalizing late -- a window bounded + // by one artifact's upload+list latency (seconds, for these KB parquet files). + // We therefore keep scanning `stopAfterSeen` artifacts past the newest + // already-ingested one as a cushion; any new (un-ingested) artifact resets the + // counter. To bury a late finalizer we'd need `stopAfterSeen` newer artifacts + // ingested above it while it uploads, i.e. an upload outlasting a whole cron + // interval -- impossible here, so this misses nothing in practice. + // + // `lookbackDays` is the absolute backstop for the first run, when nothing is + // ingested yet and the cushion never triggers. + async listArtifacts(prefix: string, options: ListOptions): Promise { + const { ingested, lookbackDays, stopAfterSeen } = options; + const cutoff = Date.now() - lookbackDays * 24 * 60 * 60 * 1000; + const out: Artifact[] = []; + let seen = 0; + for await (const artifact of this._paginateArtifacts('/actions/artifacts?per_page=100')) { + const createdAt = artifact.created_at ? Date.parse(artifact.created_at) : 0; + if (createdAt && createdAt < cutoff) + return out; + if (artifact.expired || !artifact.name.startsWith(prefix)) + continue; + const id = String(artifact.id); + if (ingested.has(id)) { + if (++seen >= stopAfterSeen) + return out; + continue; + } + seen = 0; + out.push({ id, name: artifact.name }); + } + return out; + } + + // The newest non-expired artifact with the exact name, or null if none. + async findLatestArtifact(name: string): Promise { + const query = `/actions/artifacts?name=${encodeURIComponent(name)}&per_page=100`; + for await (const artifact of this._paginateArtifacts(query)) { + if (!artifact.expired) + return String(artifact.id); + } + return null; + } + + async downloadArtifactZip(id: string): Promise { + // 302 -> blob storage; fetch follows it and strips the Authorization header + // on the cross-origin redirect, as required by the signed URL. + const response = await fetch(`${this._base}/actions/artifacts/${id}/zip`, { headers: this._headers }); + if (!response.ok) + throw new Error(`Failed to download artifact ${id}: ${response.status} ${response.statusText}`); + return Buffer.from(await response.arrayBuffer()); + } + + private async * _paginateArtifacts(path: string): AsyncGenerator { + let url: string | null = `${this._base}${path}`; + while (url) { + const response = await fetch(url, { headers: this._headers }); + if (!response.ok) + throw new Error(`GitHub API error: ${response.status} ${response.statusText} for ${url}`); + const body = await response.json() as { artifacts?: RawArtifact[] }; + for (const artifact of body.artifacts ?? []) + yield artifact; + url = nextPageUrl(response.headers.get('link')); + } + } +} + +// Parse the `rel="next"` target out of a GitHub Link header, or null if absent. +function nextPageUrl(link: string | null): string | null { + if (!link) + return null; + for (const part of link.split(',')) { + const match = part.match(/<([^>]+)>\s*;\s*rel="next"/); + if (match) + return match[1]; + } + return null; +} + +// Extract the first zip entry whose name ends with `ext` to `destPath`. +export async function extractSingle(zipBuffer: Buffer, ext: string, destPath: string): Promise { + return new Promise((resolve, reject) => { + yauzl.fromBuffer(zipBuffer, { lazyEntries: true }, (error, zipFile) => { + if (error || !zipFile) { + reject(error ?? new Error('Failed to open zip')); + return; + } + let found = false; + zipFile.on('entry', entry => { + if (/\/$/.test(entry.fileName) || !entry.fileName.endsWith(ext)) { + zipFile.readEntry(); + return; + } + found = true; + zipFile.openReadStream(entry, (streamError, stream) => { + if (streamError || !stream) { + reject(streamError ?? new Error('Failed to read zip entry')); + return; + } + const out = fs.createWriteStream(destPath); + stream.on('error', reject); + out.on('error', reject); + out.on('finish', () => resolve(destPath)); + stream.pipe(out); + }); + }); + zipFile.on('end', () => { + if (!found) + reject(new Error(`No "*${ext}" entry found in artifact zip`)); + }); + zipFile.on('error', reject); + zipFile.readEntry(); + }); + }); +} + +// Split `items` into consecutive batches of at most `size`. +export function chunk(items: T[], size: number): T[][] { + const batches: T[][] = []; + for (let i = 0; i < items.length; i += size) + batches.push(items.slice(i, i + size)); + return batches; +} diff --git a/utils/test-results-db/package.json b/utils/test-results-db/package.json new file mode 100644 index 0000000000000..3dbc1ca591c05 --- /dev/null +++ b/utils/test-results-db/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/utils/test-results-db/truncate.ts b/utils/test-results-db/truncate.ts new file mode 100644 index 0000000000000..b15c073d7fa62 --- /dev/null +++ b/utils/test-results-db/truncate.ts @@ -0,0 +1,32 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TestResultsDb, fileSize, formatBytes } from './db.ts'; + +// Keep only the newest `maxRuns` runs (a run is a `(run_id, run_attempt)` pair), +// delete the rest, and compact to reclaim the freed disk space. +export async function cmdTruncate(dbPath: string, maxRuns: number): Promise { + const db = await TestResultsDb.open(dbPath); + try { + const before = await db.runCount(); + await db.truncateToRuns(maxRuns); + const after = await db.runCount(); + console.log(`Truncated ${before} -> ${after} runs (cap ${maxRuns})`); + console.log(` ${await db.rowCount()} rows, size ${formatBytes(fileSize(dbPath))}`); + } finally { + db.close(); + } +} diff --git a/utils/test-results-db/update.ts b/utils/test-results-db/update.ts new file mode 100644 index 0000000000000..52e732633b389 --- /dev/null +++ b/utils/test-results-db/update.ts @@ -0,0 +1,81 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { TestResultsDb, fileSize, formatBytes } from './db.ts'; +import { GitHubClient, chunk, extractSingle } from './github.ts'; + +const PARQUET_ARTIFACT_PREFIX = 'parquet-report-'; + +export type UpdateOptions = { + lookbackDays: number; + concurrency: number; + stopAfterSeen: number; +}; + +// Ingest the parquet artifacts that aren't in the database yet. Downloads run +// `concurrency`-wide in batches (network-bound, the slow part); each batch is +// then ingested serially on the single connection and its temp files removed, +// so disk usage stays bounded to one batch. +export async function cmdUpdate(dbPath: string, token: string, options: UpdateOptions): Promise { + const github = new GitHubClient(token); + const db = await TestResultsDb.open(dbPath); + try { + const ingested = await db.ingestedArtifactIds(); + console.log(`Test results database`); + console.log(` ${await db.rowCount()} rows from ${ingested.size} artifacts`); + + const todo = await github.listArtifacts(PARQUET_ARTIFACT_PREFIX, { + ingested, + lookbackDays: options.lookbackDays, + stopAfterSeen: options.stopAfterSeen, + }); + console.log(`\nScanning for new artifacts (last ${options.lookbackDays} days)`); + console.log(` ${todo.length} new artifacts to import`); + + let imported = 0; + for (const batch of chunk(todo, options.concurrency)) { + const files = await Promise.all(batch.map(async artifact => { + const zip = await github.downloadArtifactZip(artifact.id); + const file = path.join(os.tmpdir(), `trdb-${artifact.id}.parquet`); + await extractSingle(zip, '.parquet', file); + return { id: artifact.id, file }; + })); + for (const { id, file } of files) { + try { + await db.ingestParquet(file, id); + imported++; + } finally { + fs.rmSync(file, { force: true }); + } + } + console.log(` imported ${imported}/${todo.length}`); + } + + console.log(`\nSummary`); + console.log(` imported ${imported} new artifacts`); + console.log(` ${await db.rowCount()} rows from ${await db.runCount()} runs`); + console.log(` size ${formatBytes(fileSize(dbPath))}`); + + if (process.env.GITHUB_OUTPUT) + fs.appendFileSync(process.env.GITHUB_OUTPUT, `imported=${imported}\n`); + } finally { + db.close(); + } +}