diff --git a/gate-engine/review/__tests__/mine-telemetry.test.mts b/gate-engine/review/__tests__/mine-telemetry.test.mts new file mode 100644 index 0000000..97abfdd --- /dev/null +++ b/gate-engine/review/__tests__/mine-telemetry.test.mts @@ -0,0 +1,608 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; +import { collectRepoArgs, sqlite3Available, sqliteJson } from '../eval/reviewers/mine-common.mts'; +import { + buildFailFixCandidate, + buildHistogram, + buildWaivedCandidate, + diffArchiveRelPath, + findNextLensOutcome, + findNextOutcome, + groupShipsByRepoBranch, + hasDiffEvidence, + histogramKey, + INLINE_DIFF_CAP_BYTES, + isSameDiff, + mergeCandidates, + pickFailReason, + resolveDiffPayload, + selectFailLensRows, + sortByTsStart, + telemetryUrl, +} from '../eval/reviewers/mine-telemetry-lib.mts'; + +describe('mine-telemetry-lib: sortByTsStart', () => { + it('sorts ascending by ts_start', () => { + const ships = [ + { ship_id: 'b', ts_start: '2026-08-01T22:11:51.523Z' }, + { ship_id: 'a', ts_start: '2026-08-01T22:02:46.951Z' }, + { ship_id: 'c', ts_start: '2026-08-01T22:27:21.171Z' }, + ]; + expect(sortByTsStart(ships).map((s) => s.ship_id)).toEqual(['a', 'b', 'c']); + }); + + it('tiebreaks equal timestamps by ship_id for determinism', () => { + const ships = [ + { ship_id: 'zeta', ts_start: '2026-08-01T22:00:00.000Z' }, + { ship_id: 'alpha', ts_start: '2026-08-01T22:00:00.000Z' }, + ]; + expect(sortByTsStart(ships).map((s) => s.ship_id)).toEqual(['alpha', 'zeta']); + }); + + it('does not mutate the input array', () => { + const ships = [ + { ship_id: 'b', ts_start: '2026-08-01T22:11:51.523Z' }, + { ship_id: 'a', ts_start: '2026-08-01T22:02:46.951Z' }, + ]; + const copy = [...ships]; + sortByTsStart(ships); + expect(ships).toEqual(copy); + }); + + it('orders a same-second millis vs no-millis tie chronologically (not lexicographically)', () => { + // Plain string compare would put the no-millis form ("...Z") AFTER the millis form + // ("...523Z") at this exact second, because '.' < 'Z' — inverting true chronology, since + // the no-millis form is chronologically the earlier moment (implicit .000) of that second. + const ships = [ + { ship_id: 'later', ts_start: '2026-08-01T22:11:51.523Z' }, + { ship_id: 'earlier', ts_start: '2026-08-01T22:11:51Z' }, + ]; + expect(sortByTsStart(ships).map((s) => s.ship_id)).toEqual(['earlier', 'later']); + }); + + it('falls back to string ordering when a timestamp fails to parse, without throwing', () => { + const ships = [ + { ship_id: 'b', ts_start: 'not-a-date' }, + { ship_id: 'a', ts_start: '2026-08-01T22:00:00Z' }, + ]; + expect(() => sortByTsStart(ships)).not.toThrow(); + expect( + sortByTsStart(ships) + .map((s) => s.ship_id) + .sort(), + ).toEqual(['a', 'b']); + }); +}); + +describe('mine-telemetry-lib: groupShipsByRepoBranch', () => { + const ships = [ + { + ship_id: 's1', + repo: 'devkit', + branch: 'bench/waive-command', + ts_start: '2026-08-01T22:02:46Z', + }, + { + ship_id: 's2', + repo: 'devkit', + branch: 'bench/waive-command', + ts_start: '2026-08-01T22:05:26Z', + }, + { ship_id: 's3', repo: 'devkit', branch: 'main', ts_start: '2026-08-01T22:06:00Z' }, + { ship_id: 's4', repo: 'devkit', branch: null, ts_start: '2026-08-01T22:07:00Z' }, + { ship_id: 's5', repo: 'devkit', branch: '', ts_start: '2026-08-01T22:08:00Z' }, + ]; + + it('groups by repo+branch and sorts each chain by ts_start', () => { + const { groups } = groupShipsByRepoBranch(ships); + const chain = groups.get('devkit::bench/waive-command'); + expect(chain.map((s) => s.ship_id)).toEqual(['s1', 's2']); + expect(groups.get('devkit::main').map((s) => s.ship_id)).toEqual(['s3']); + }); + + it('excludes null/blank branch ships and counts them', () => { + const { groups, skippedNullBranch } = groupShipsByRepoBranch(ships); + expect(skippedNullBranch).toBe(2); + for (const chain of groups.values()) { + expect(chain.some((s) => s.ship_id === 's4' || s.ship_id === 's5')).toBe(false); + } + }); + + it('builds an indexByShipId lookup consistent with each chain', () => { + const { groups, indexByShipId } = groupShipsByRepoBranch(ships); + const loc = indexByShipId.get('s2'); + expect(loc).toEqual({ key: 'devkit::bench/waive-command', idx: 1 }); + expect(groups.get(loc.key)[loc.idx].ship_id).toBe('s2'); + }); + + it('returns empty groups/zero skips for an empty input', () => { + const { groups, indexByShipId, skippedNullBranch } = groupShipsByRepoBranch([]); + expect(groups.size).toBe(0); + expect(indexByShipId.size).toBe(0); + expect(skippedNullBranch).toBe(0); + }); +}); + +describe('mine-telemetry-lib: findNextOutcome', () => { + // Mirrors the live 5-ship devkit/bench-waive-command sequence from the scout report. + const chain = [ + { ship_id: 's0', exit_code: 1 }, // blocked=deterministic, never reached review + { ship_id: 's1', exit_code: 1 }, // FAIL (the anchor) + { ship_id: 's2', exit_code: 1 }, // FAIL again — still broken + { ship_id: 's3', exit_code: 1 }, // FAIL again + { ship_id: 's4', exit_code: 0 }, // PASS — the fix + ]; + + it('scans past consecutive fails to find the eventual pass (does not stop at the very next ship)', () => { + const statusOf = (shipId) => ({ s2: 'fail', s3: 'fail', s4: 'pass' })[shipId]; + expect(findNextOutcome(chain, 's1', 'correctness-reviewer', statusOf)).toEqual({ + kind: 'fixed', + nextShipId: 's4', + }); + }); + + it('stops immediately when the very next ship already passes', () => { + const statusOf = (shipId) => ({ s2: 'pass' })[shipId]; + expect(findNextOutcome(chain, 's1', 'correctness-reviewer', statusOf)).toEqual({ + kind: 'fixed', + nextShipId: 's2', + }); + }); + + it('treats an absent reviewer on a clean ship as fixed-by-absence', () => { + const statusOf = () => undefined; // reviewer never appears again + expect(findNextOutcome(chain, 's3', 'correctness-reviewer', statusOf)).toEqual({ + kind: 'fixed', + nextShipId: 's4', + }); + }); + + it('keeps scanning when the reviewer is absent and the ship did not ship clean', () => { + const blockedChain = [ + { ship_id: 's1', exit_code: 1 }, + { ship_id: 's2', exit_code: 1 }, // absent + blocked → no signal + { ship_id: 's3', exit_code: 0 }, // absent + clean → fixed-by-absence + ]; + const statusOf = () => undefined; + expect(findNextOutcome(blockedChain, 's1', 'r', statusOf)).toEqual({ + kind: 'fixed', + nextShipId: 's3', + }); + }); + + it('returns no-fix-found when the chain ends without a pass', () => { + const statusOf = () => 'fail'; + expect(findNextOutcome(chain, 's1', 'correctness-reviewer', statusOf)).toEqual({ + kind: 'no-fix-found', + nextShipId: null, + }); + }); + + it('returns no-fix-found when the fail ship is the last in its chain (branch abandonment)', () => { + const statusOf = () => undefined; + expect(findNextOutcome(chain, 's4', 'correctness-reviewer', statusOf)).toEqual({ + kind: 'no-fix-found', + nextShipId: null, + }); + }); + + it('returns no-fix-found when the anchor ship is not found in the chain', () => { + expect(findNextOutcome(chain, 'does-not-exist', 'r', () => 'pass')).toEqual({ + kind: 'no-fix-found', + nextShipId: null, + }); + }); +}); + +describe('mine-telemetry-lib: findNextLensOutcome', () => { + // Reproduces the reported blocker: a reviewer FAILs the same ship on TWO distinct lenses, and + // only one of them actually gets fixed by the next ship. A reviewer-level scan (findNextOutcome) + // would wrongly report both lenses fixed once the reviewer's overall status flips to pass. + const chain = [ + { ship_id: 's1', exit_code: 1 }, // FAIL on both lens-a and lens-b (the anchor) + { ship_id: 's2', exit_code: 0 }, // lens-a passes; lens-b has no lens-level row at all here, + // but the reviewer overall also passes (unrelated files touched this time). + ]; + + it('reports fixed for the lens that has its own pass row', () => { + const lensStatusOf = (shipId, reviewer, lens) => + ({ 's2::r::lens-a': 'pass' })[`${shipId}::${reviewer}::${lens}`]; + const statusOf = () => 'pass'; + expect(findNextLensOutcome(chain, 's1', 'r', 'lens-a', statusOf, lensStatusOf)).toEqual({ + kind: 'fixed', + nextShipId: 's2', + }); + }); + + it('does not blindly copy the reviewer-level pass onto a lens still reported failing', () => { + const failingChain = [ + { ship_id: 's1', exit_code: 1 }, + { ship_id: 's2', exit_code: 0 }, // reviewer overall passes... + ]; + const lensStatusOf = (shipId, reviewer, lens) => + ({ 's2::r::lens-b': 'fail' })[`${shipId}::${reviewer}::${lens}`]; // ...but lens-b itself still fails + const statusOf = () => 'pass'; + expect(findNextLensOutcome(failingChain, 's1', 'r', 'lens-b', statusOf, lensStatusOf)).toEqual({ + kind: 'no-fix-found', + nextShipId: null, + }); + }); + + it('falls back to reviewer-level pass when the candidate ship has no lens-level row for this lens', () => { + const lensStatusOf = () => undefined; // no lens breakdown recorded on the candidate ship + const statusOf = () => 'pass'; + expect(findNextLensOutcome(chain, 's1', 'r', 'lens-b', statusOf, lensStatusOf)).toEqual({ + kind: 'fixed', + nextShipId: 's2', + }); + }); + + it('keeps scanning (does not assume fixed) when lens data is absent and the reviewer still fails overall', () => { + const longerChain = [ + { ship_id: 's1', exit_code: 1 }, + { ship_id: 's2', exit_code: 1 }, // reviewer still fails overall (maybe on the OTHER lens); + // no lens-level row for lens-b here — must not assume lens-b resolved. + { ship_id: 's3', exit_code: 0 }, + ]; + const lensStatusOf = (shipId) => ({ s3: 'pass' })[shipId] && 'pass'; + const statusOf = (shipId) => ({ s2: 'fail', s3: 'pass' })[shipId]; + expect(findNextLensOutcome(longerChain, 's1', 'r', 'lens-b', statusOf, lensStatusOf)).toEqual({ + kind: 'fixed', + nextShipId: 's3', + }); + }); + + it('returns no-fix-found when the anchor ship is not found in the chain', () => { + expect( + findNextLensOutcome( + chain, + 'missing', + 'r', + 'lens-a', + () => 'pass', + () => 'pass', + ), + ).toEqual({ kind: 'no-fix-found', nextShipId: null }); + }); +}); + +describe('mine-telemetry-lib: hasDiffEvidence', () => { + it('is true when either hash is present', () => { + expect(hasDiffEvidence('abc', null)).toBe(true); + expect(hasDiffEvidence(null, 'def')).toBe(true); + expect(hasDiffEvidence('abc', 'def')).toBe(true); + }); + + it('is false when both hashes are null', () => { + expect(hasDiffEvidence(null, null)).toBe(false); + }); +}); + +describe('mine-telemetry-lib: isSameDiff', () => { + it('is true only for two equal non-empty hashes', () => { + expect(isSameDiff('abc', 'abc')).toBe(true); + }); + + it('is false for different hashes', () => { + expect(isSameDiff('abc', 'def')).toBe(false); + }); + + it('is false when either side is null/undefined/empty', () => { + expect(isSameDiff(null, 'abc')).toBe(false); + expect(isSameDiff('abc', undefined)).toBe(false); + expect(isSameDiff('', '')).toBe(false); + }); +}); + +describe('mine-telemetry-lib: diffArchiveRelPath', () => { + it('builds the diffs/.diff.gz relative path', () => { + expect(diffArchiveRelPath('deadbeef')).toBe('diffs/deadbeef.diff.gz'); + }); + + it('is null for a missing hash', () => { + expect(diffArchiveRelPath(null)).toBeNull(); + expect(diffArchiveRelPath(undefined)).toBeNull(); + }); +}); + +describe('mine-telemetry-lib: resolveDiffPayload', () => { + it('inlines text at or under the cap', () => { + const text = 'x'.repeat(100); + expect(resolveDiffPayload(text, 'diffs/h.diff.gz')).toEqual({ diffText: text, diffPath: null }); + }); + + it('inlines text exactly at the cap boundary', () => { + const text = 'x'.repeat(INLINE_DIFF_CAP_BYTES); + expect(resolveDiffPayload(text, 'diffs/h.diff.gz')).toEqual({ diffText: text, diffPath: null }); + }); + + it('falls back to a path ref past the cap', () => { + const text = 'x'.repeat(INLINE_DIFF_CAP_BYTES + 1); + expect(resolveDiffPayload(text, 'diffs/h.diff.gz')).toEqual({ + diffText: null, + diffPath: 'diffs/h.diff.gz', + }); + }); + + it('returns both null when there is no archived text', () => { + expect(resolveDiffPayload(null, 'diffs/h.diff.gz')).toEqual({ diffText: null, diffPath: null }); + }); +}); + +describe('mine-telemetry-lib: telemetryUrl', () => { + it('includes the lens segment when present', () => { + expect(telemetryUrl('fail-fix', 'ship1', 'correctness-reviewer', 'concurrency-races')).toBe( + 'telemetry://fail-fix/ship1/correctness-reviewer/concurrency-races', + ); + }); + + it('omits the lens segment when absent', () => { + expect(telemetryUrl('waived-decoy', 'ship1', 'correctness-reviewer', null)).toBe( + 'telemetry://waived-decoy/ship1/correctness-reviewer', + ); + }); +}); + +describe('mine-telemetry-lib: pickFailReason', () => { + it('prefers the lens issues_json text when parseable', () => { + expect(pickFailReason('["finding one","finding two"]', 'fallback reason')).toBe( + 'finding one\nfinding two', + ); + }); + + it('falls back to the review reason when issues_json is missing', () => { + expect(pickFailReason(null, 'fallback reason')).toBe('fallback reason'); + }); + + it('falls back to the review reason when issues_json is unparseable', () => { + expect(pickFailReason('not json', 'fallback reason')).toBe('fallback reason'); + }); + + it('falls back to the review reason when issues_json parses to an empty array', () => { + expect(pickFailReason('[]', 'fallback reason')).toBe('fallback reason'); + }); + + it('returns null when neither source has usable text', () => { + expect(pickFailReason(null, null)).toBeNull(); + expect(pickFailReason('[]', ' ')).toBeNull(); + }); +}); + +describe('mine-telemetry-lib: buildFailFixCandidate', () => { + it('shapes a full fail-fix row', () => { + const row = buildFailFixCandidate({ + shipId: 's1', + repo: 'devkit', + branch: 'bench/waive-command', + reviewer: 'correctness-reviewer', + lens: 'concurrency-races', + tsFail: '2026-08-01T22:11:51.523Z', + diffSha256: 'aaa', + bytesAvailable: true, + diffPayload: { diffText: 'diff --git a b', diffPath: null }, + failReason: 'concurrency race', + nextShipId: 's4', + nextDiffSha256: 'bbb', + nextBytesAvailable: false, + nextDiffPayload: { diffText: null, diffPath: null }, + tsFix: '2026-08-01T22:27:21.171Z', + }); + expect(row).toMatchObject({ + kind: 'fail-fix', + url: 'telemetry://fail-fix/s1/correctness-reviewer/concurrency-races', + shipId: 's1', + diffSha256: 'aaa', + bytesAvailable: true, + diffText: 'diff --git a b', + nextShipId: 's4', + nextDiffSha256: 'bbb', + nextBytesAvailable: false, + hasDiffEvidence: true, + }); + }); + + it('defaults missing diff payload/hash fields to null rather than throwing, and flags hasDiffEvidence false', () => { + const row = buildFailFixCandidate({ + shipId: 's1', + repo: 'devkit', + branch: 'main', + reviewer: 'correctness-reviewer', + lens: null, + tsFail: 't1', + diffSha256: null, + bytesAvailable: false, + diffPayload: undefined, + failReason: null, + nextShipId: 's2', + nextDiffSha256: null, + nextBytesAvailable: false, + nextDiffPayload: undefined, + tsFix: 't2', + }); + expect(row.diffText).toBeNull(); + expect(row.diffPath).toBeNull(); + expect(row.nextDiffText).toBeNull(); + expect(row.lens).toBeNull(); + expect(row.hasDiffEvidence).toBe(false); + }); +}); + +describe('mine-telemetry-lib: buildWaivedCandidate', () => { + it('shapes a waived-decoy row with a null rationale by default', () => { + const row = buildWaivedCandidate({ + shipId: 'E593DD18', + repo: 'devkit', + branch: 'main', + reviewer: 'correctness-reviewer', + lens: 'writer-reader-contracts', + tsFail: '2026-08-01T00:00:00Z', + diffSha256: 'ccc', + bytesAvailable: false, + diffPayload: { diffText: null, diffPath: null }, + failReason: 'the judge finding text', + disposition: 'waived', + }); + expect(row).toMatchObject({ + kind: 'waived-decoy', + url: 'telemetry://waived-decoy/E593DD18/correctness-reviewer/writer-reader-contracts', + disposition: 'waived', + rationale: null, + }); + }); +}); + +describe('mine-telemetry-lib: mergeCandidates', () => { + it('new rows win by url; untouched existing rows are preserved', () => { + const existing = new Map([ + ['telemetry://a/r', { url: 'telemetry://a/r', tsFail: 'old' }], + ['telemetry://b/r', { url: 'telemetry://b/r', tsFail: 'keep' }], + ]); + const merged = mergeCandidates(existing, [{ url: 'telemetry://a/r', tsFail: 'new' }]); + expect(merged.get('telemetry://a/r').tsFail).toBe('new'); + expect(merged.get('telemetry://b/r').tsFail).toBe('keep'); + expect(merged.size).toBe(2); + }); + + it('ignores rows without a url', () => { + const merged = mergeCandidates(new Map(), [{ tsFail: 'no-url' }]); + expect(merged.size).toBe(0); + }); + + it('tolerates an undefined existing map', () => { + const merged = mergeCandidates(undefined, [{ url: 'telemetry://a/r' }]); + expect(merged.size).toBe(1); + }); +}); + +describe('mine-telemetry-lib: histogramKey / buildHistogram', () => { + it('groups by kind, reviewer, and bytesAvailable', () => { + const rows = [ + { kind: 'fail-fix', reviewer: 'correctness-reviewer', bytesAvailable: true }, + { kind: 'fail-fix', reviewer: 'correctness-reviewer', bytesAvailable: true }, + { kind: 'fail-fix', reviewer: 'correctness-reviewer', bytesAvailable: false }, + { kind: 'waived-decoy', reviewer: 'correctness-reviewer', bytesAvailable: false }, + ]; + const hist = buildHistogram(rows, histogramKey); + expect(hist[0]).toEqual(['fail-fix / correctness-reviewer / bytesAvailable=true', 2]); + expect(hist).toContainEqual(['fail-fix / correctness-reviewer / bytesAvailable=false', 1]); + expect(hist).toContainEqual(['waived-decoy / correctness-reviewer / bytesAvailable=false', 1]); + }); + + it('returns an empty histogram for no rows', () => { + expect(buildHistogram([])).toEqual([]); + }); +}); + +describe('mine-telemetry-lib: selectFailLensRows', () => { + const lens = (l, status, disposition) => ({ lens: l, status, disposition }); + + it('returns the blocking failed lenses when the breakdown has them', () => { + const { rows, skipped } = selectFailLensRows([ + lens('races', 'fail', 'blocking'), + lens('contracts', 'pass', null), + ]); + expect(rows.map((r) => r.lens)).toEqual(['races']); + expect(skipped).toBeNull(); + }); + + it('treats a null disposition (pre-disposition-era row) as blocking', () => { + const { rows, skipped } = selectFailLensRows([lens('races', 'fail', null)]); + expect(rows.map((r) => r.lens)).toEqual(['races']); + expect(skipped).toBeNull(); + }); + + it('falls back to the reviewer-level scan when no lens breakdown was recorded', () => { + for (const absent of [undefined, null, []]) { + const { rows, skipped } = selectFailLensRows(absent); + expect(rows).toEqual([null]); + expect(skipped).toBeNull(); + } + }); + + // The regression this guards: an empty blocking filter used to fall through to [null], so a + // fail whose only failing lenses were waived/dropped minted a reviewer-level gold from exactly + // the lenses the allowlist had just excluded. + it('skips the fail entirely when every failing lens is waived or dropped', () => { + for (const disposition of ['waived', 'dropped_out_of_charter']) { + const { rows, skipped } = selectFailLensRows([ + lens('races', 'fail', disposition), + lens('contracts', 'pass', null), + ]); + expect(rows).toEqual([]); + expect(skipped).toBe('all-failing-lenses-non-blocking'); + } + }); + + it('skips when a mix of waived and dropped failing lenses leaves nothing blocking', () => { + const { rows, skipped } = selectFailLensRows([ + lens('races', 'fail', 'waived'), + lens('contracts', 'fail', 'dropped_out_of_charter'), + ]); + expect(rows).toEqual([]); + expect(skipped).toBe('all-failing-lenses-non-blocking'); + }); + + it('still falls back to reviewer-level when a breakdown exists but nothing failed in it', () => { + const { rows, skipped } = selectFailLensRows([lens('races', 'pass', null)]); + expect(rows).toEqual([null]); + expect(skipped).toBeNull(); + }); +}); + +describe('mine-common: collectRepoArgs', () => { + it('collects every repeated --repo value', () => { + expect(collectRepoArgs(['--dev', '--repo', 'devkit', '--repo', 'frink'])).toEqual([ + 'devkit', + 'frink', + ]); + }); + + it('returns an empty list when --repo is absent (callers then use their defaults)', () => { + expect(collectRepoArgs(['--dev', '--max', '20'])).toEqual([]); + }); + + // Both callers treat a non-empty result as an EXPLICIT scope replacing their defaults, so + // swallowing the next flag here would silently mine a repo that cannot exist. + it('rejects a flag-shaped value instead of storing it as a repository', () => { + expect(() => collectRepoArgs(['--repo', '--dev'])).toThrow(/--repo needs a repository name/); + }); + + it('rejects a trailing --repo with no value', () => { + expect(() => collectRepoArgs(['--dev', '--repo'])).toThrow(/--repo needs a repository name/); + }); +}); + +describe('mine-common: sqliteJson', () => { + const tmp = mkdtempSync(path.join(os.tmpdir(), 'mine-common-sqlite-')); + const dbPath = path.join(tmp, 'usage.db'); + afterAll(() => rmSync(tmp, { recursive: true, force: true })); + + const available = sqlite3Available(); + const maybe = available ? it : it.skip; + + if (available) { + execFileSync('sqlite3', [dbPath, 'CREATE TABLE t(a); INSERT INTO t VALUES(1);']); + } + + maybe('reads rows as parsed JSON', () => { + expect(sqliteJson(dbPath, 'SELECT a FROM t;')).toEqual([{ a: 1 }]); + }); + + maybe('returns an empty array for an empty result set', () => { + expect(sqliteJson(dbPath, 'SELECT a FROM t WHERE a = 99;')).toEqual([]); + }); + + // The read-only boundary is enforced by SQLite itself (-readonly), not by convention: the + // miners are strictly read-side and the collector owns every write path. + maybe('refuses a write and leaves the database untouched', () => { + expect(() => sqliteJson(dbPath, 'INSERT INTO t VALUES(2);')).toThrow(); + expect(sqliteJson(dbPath, 'SELECT a FROM t;')).toEqual([{ a: 1 }]); + }); + + maybe('refuses DDL as well', () => { + expect(() => sqliteJson(dbPath, 'CREATE TABLE evil(x);')).toThrow(); + }); +}); diff --git a/gate-engine/review/eval/reviewers/mine-bots.mts b/gate-engine/review/eval/reviewers/mine-bots.mts index bdf14cb..21dc391 100644 --- a/gate-engine/review/eval/reviewers/mine-bots.mts +++ b/gate-engine/review/eval/reviewers/mine-bots.mts @@ -36,6 +36,12 @@ import { parseCoderabbitMarker, sqlString, } from './mine-bots-lib.mts'; +import { + collectRepoArgs, + readCandidatesFile, + sqlite3Available, + sqliteJson, +} from './mine-common.mts'; const here = path.dirname(fileURLToPath(import.meta.url)); const OUT = path.join(here, 'candidates.jsonl'); @@ -221,19 +227,10 @@ function commitsAfter(repo, prCommits, afterIso, fileCache) { function resolveScopeDb() { const dbPath = process.env.USAGE_DB || path.join(os.homedir(), '.claude-usage', 'usage.db'); if (!existsSync(dbPath)) return null; - try { - execFileSync('sqlite3', ['-version'], { stdio: 'ignore' }); - } catch { - return null; - } + if (!sqlite3Available()) return null; return dbPath; } -function sqliteJson(dbPath, sql) { - const raw = execFileSync('sqlite3', [dbPath, '-json', sql], { encoding: 'utf8' }).trim(); - return raw ? JSON.parse(raw) : []; -} - function scopeForPr(dbPath, cache, repoFull, repoShort, prNumber) { const key = `${repoFull}#${prNumber}`; if (cache.has(key)) return cache.get(key); @@ -262,21 +259,6 @@ function scopeForPr(dbPath, cache, repoFull, repoShort, prNumber) { // Merge / dedupe against existing candidates.jsonl and the promoted corpus. // --------------------------------------------------------------------------------------------- -function readExistingCandidates() { - const map = new Map(); - if (!existsSync(OUT)) return map; - for (const line of readFileSync(OUT, 'utf8').split('\n')) { - if (!line.trim()) continue; - try { - const row = JSON.parse(line); - if (row.url) map.set(row.url, row); - } catch { - // skip malformed line rather than aborting the whole merge - } - } - return map; -} - function collectCorpusUrls(dir) { const urls = new Set(); let entries = []; @@ -312,14 +294,7 @@ function collectCorpusUrls(dir) { // Main sweep. // --------------------------------------------------------------------------------------------- -const repoArgs = []; -const argv = process.argv.slice(2); -for (let i = 0; i < argv.length; i += 1) { - if (argv[i] === '--repo' && argv[i + 1]) { - i += 1; - repoArgs.push(argv[i]); - } -} +const repoArgs = collectRepoArgs(process.argv.slice(2)); const repos = repoArgs.length > 0 ? repoArgs : DEFAULT_REPOS; const scopeDb = resolveScopeDb(); @@ -453,7 +428,7 @@ for (const repo of repos) { // Merge: new data wins by url, but rows we didn't re-sweep this run (other repos/PRs not passed // via --repo, or a PR gh failed to fetch this time) are preserved rather than dropped. -const merged = readExistingCandidates(); +const merged = readCandidatesFile(OUT); for (const row of newRows) { if (!row.url) { console.error(`mine-bots: dropping comment ${row.id} — no html_url to key on`); diff --git a/gate-engine/review/eval/reviewers/mine-common.mts b/gate-engine/review/eval/reviewers/mine-common.mts new file mode 100644 index 0000000..ee24878 --- /dev/null +++ b/gate-engine/review/eval/reviewers/mine-common.mts @@ -0,0 +1,78 @@ +// @ts-nocheck — BENCH-ONLY (excluded from tsc, see tsconfig.json exclude); loose types deliberate. + +/** + * mine-common — the plumbing both miners (mine-bots.mts, mine-telemetry.mts) share: url-keyed + * candidates-file reading, `--repo` argv collection, and read-only sqlite3 access. Extracted so + * the two stay byte-identical by construction instead of by copy (commit-guard caught the copies). + */ + +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; + +/** Parse an existing url-keyed candidates .jsonl into a Map; malformed lines are + * skipped rather than aborting the whole merge. */ +export function readCandidatesFile(file) { + const map = new Map(); + if (!existsSync(file)) return map; + for (const line of readFileSync(file, 'utf8').split('\n')) { + if (!line.trim()) continue; + try { + const row = JSON.parse(line); + if (row.url) map.set(row.url, row); + } catch { + // skip malformed line rather than aborting the whole merge + } + } + return map; +} + +/** Collect every `--repo ` pair from argv (repeatable flag, both miners). + * + * A missing or flag-shaped value is a hard usage error, never a silently-accepted repo. Both + * callers treat ANY non-empty result as an explicit scope that replaces their defaults + * (mine-bots' DEFAULT_REPOS, mine-telemetry's allowlist), so swallowing the following flag — + * `--repo --dev` storing `"--dev"` as the repository — would narrow the sweep to a repo that + * cannot exist and mine nothing, while reporting a clean run. Failing loudly is the only safe + * reading of an incomplete pair. */ +export function collectRepoArgs(argv) { + const repoArgs = []; + for (let i = 0; i < argv.length; i += 1) { + if (argv[i] !== '--repo') continue; + const value = argv[i + 1]; + if (!value || value.startsWith('--')) { + throw new Error( + `--repo needs a repository name, got ${value ? `"${value}"` : 'nothing'} — ` + + 'pass it as `--repo ` (repeatable).', + ); + } + i += 1; + repoArgs.push(value); + } + return repoArgs; +} + +/** True when the sqlite3 CLI is invocable (both miners fail open without it). */ +export function sqlite3Available() { + try { + execFileSync('sqlite3', ['-version'], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +/** Read-only -json SELECT via the sqlite3 CLI. maxBuffer sized for the largest expected result + * set (mine-telemetry sweeps whole tables; mine-bots only ever joins per-PR). + * + * `-readonly` opens the collector db in SQLite's own read-only mode, so the boundary is enforced + * by the engine rather than by convention: a caller that ever passes an INSERT/UPDATE/DDL gets + * "attempt to write a readonly database" instead of mutating the user's telemetry. The miners are + * strictly read-side (the collector owns every write path) and this keeps that true by + * construction. */ +export function sqliteJson(dbPath, sql, maxBuffer = 256 * 1024 * 1024) { + const raw = execFileSync('sqlite3', ['-readonly', '-json', dbPath, sql], { + encoding: 'utf8', + maxBuffer, + }).trim(); + return raw ? JSON.parse(raw) : []; +} diff --git a/gate-engine/review/eval/reviewers/mine-telemetry-lib.mts b/gate-engine/review/eval/reviewers/mine-telemetry-lib.mts new file mode 100644 index 0000000..6003c11 --- /dev/null +++ b/gate-engine/review/eval/reviewers/mine-telemetry-lib.mts @@ -0,0 +1,386 @@ +// @ts-nocheck — BENCH-ONLY (excluded from tsc, see tsconfig.json exclude); loose types deliberate. + +/** + * mine-telemetry-lib — pure helpers for mine-telemetry.mts, split out so the correlation and + * classification logic can be unit-tested without touching sqlite3, the filesystem, or the diff + * archive. Everything here takes plain data in, returns plain data out; no execFileSync, no fs. + * + * Mirrors mine-bots-lib.mts's split (network/db-free helpers vs the orchestrating script) for the + * same reason: the correlation heuristics are the part worth pinning down with fixtures. + */ + +// --------------------------------------------------------------------------------------------- +// Constants shared with the orchestrator. +// --------------------------------------------------------------------------------------------- + +// Inline the archived diff text up to this many raw (decompressed) bytes; past it, emit a +// diffPath ref instead of bloating candidates-telemetry.jsonl with megabyte-scale rows. +export const INLINE_DIFF_CAP_BYTES = 200 * 1024; + +// --------------------------------------------------------------------------------------------- +// Ship ordering / branch grouping. +// --------------------------------------------------------------------------------------------- + +// ts_start is ISO-8601, but real rows mix a with-milliseconds form ("...:00.523Z") and a +// without-milliseconds form ("...:00Z") within the same chain. Plain string comparison is NOT +// safe for that mix: at a same-second tie, "." (0x2E) sorts before "Z" (0x5A), so the no-millis +// form (implicit .000) sorts AFTER any millis form of that same second — inverting true +// chronological order. Parse both sides to epoch millis instead; only fall back to the raw +// string compare when a value fails to parse, so malformed timestamps degrade gracefully rather +// than crash or silently coerce to NaN-driven reordering. +function tsMillis(ts) { + const ms = Date.parse(String(ts ?? '')); + return Number.isNaN(ms) ? null : ms; +} + +export function sortByTsStart(ships) { + return [...(ships ?? [])].sort((a, b) => { + const ta = tsMillis(a?.ts_start); + const tb = tsMillis(b?.ts_start); + if (ta !== null && tb !== null) { + if (ta !== tb) return ta - tb; + } else { + const sa = String(a?.ts_start ?? ''); + const sb = String(b?.ts_start ?? ''); + if (sa !== sb) return sa < sb ? -1 : 1; + } + // Stable-ish tiebreak on ship_id so equal timestamps don't reorder nondeterministically + // across runs (sqlite3 -json row order isn't guaranteed for ties). + return String(a?.ship_id ?? '').localeCompare(String(b?.ship_id ?? '')); + }); +} + +const branchKey = (repo, branch) => `${repo ?? ''}::${branch}`; + +/** + * Groups ships into per-(repo,branch) chronological chains. Ships with a null/blank branch are + * excluded (uncorrelatable — see docs/benchmarks/corpus-growth.md capture-point-1 addendum) and + * counted separately rather than silently dropped. + * + * Returns `groups`: Map<"repo\0branch", ShipRow[]> sorted ascending by ts_start, and + * `indexByShipId`: Map for O(1) "where is this ship in its chain" lookups. + */ +export function groupShipsByRepoBranch(ships) { + const byKey = new Map(); + let skippedNullBranch = 0; + for (const ship of ships ?? []) { + const branch = ship?.branch; + if (branch === null || branch === undefined || String(branch).trim() === '') { + skippedNullBranch += 1; + continue; + } + const key = branchKey(ship.repo, branch); + if (!byKey.has(key)) byKey.set(key, []); + byKey.get(key).push(ship); + } + const groups = new Map(); + const indexByShipId = new Map(); + for (const [key, rows] of byKey) { + const sorted = sortByTsStart(rows); + groups.set(key, sorted); + sorted.forEach((ship, idx) => { + if (ship?.ship_id) indexByShipId.set(ship.ship_id, { key, idx }); + }); + } + return { groups, indexByShipId, skippedNullBranch }; +} + +// --------------------------------------------------------------------------------------------- +// Fail → fix correlation. +// --------------------------------------------------------------------------------------------- + +/** + * Forward-scans a branch's chronologically-sorted ships, starting just after `failShipId`, + * looking for the reviewer's fix. `statusOf(shipId, reviewer)` returns 'pass' | 'fail' | undefined + * (undefined = the reviewer didn't run / wasn't recorded for that ship — e.g. its domain wasn't + * touched, or it's absent from a deterministic-blocked ship that never reached review). + * + * Rules (see corpus-growth.md capture-point-1 addendum, verified against a live 5-ship sequence): + * - reviewer status 'fail' on a candidate ship → still broken, keep scanning (do NOT stop at the + * very next ship blindly — a retry can fail the same lens repeatedly before it lands). + * - reviewer status 'pass' → FIXED, stop here. + * - reviewer absent AND the ship shipped clean (exit_code === 0) → treat as fixed-by-absence: the + * ship reached a non-blocked end without this reviewer objecting (its scope may simply have + * moved on). This is the "absent-from-failing" case the task brief calls out. + * - reviewer absent AND the ship did NOT ship clean (blocked by something else, or still mid-flight) + * → no signal either way, keep scanning. + * - chain exhausted with no fix found → 'no-fix-found' (covers both "still failing at the end of + * the chain" and "branch abandoned" — corpus-growth.md's case (a)/(c), neither is a gold). + */ +export function findNextOutcome(shipsForBranch, failShipId, reviewer, statusOf) { + const chain = shipsForBranch ?? []; + const failIdx = chain.findIndex((s) => s?.ship_id === failShipId); + if (failIdx === -1) return { kind: 'no-fix-found', nextShipId: null }; + for (let i = failIdx + 1; i < chain.length; i += 1) { + const candidate = chain[i]; + const status = statusOf(candidate.ship_id, reviewer); + if (status === 'pass') return { kind: 'fixed', nextShipId: candidate.ship_id }; + if (status === 'fail') continue; // still broken — keep scanning + // status is undefined (reviewer absent from this ship's commit_reviews rows) + if (candidate.exit_code === 0) return { kind: 'fixed', nextShipId: candidate.ship_id }; + // absent and not clean — no signal, keep scanning + } + return { kind: 'no-fix-found', nextShipId: null }; +} + +/** + * Lens-specific variant of findNextOutcome. `statusOf` is reviewer-level (as above); + * `lensStatusOf(shipId, reviewer, lens)` returns 'pass' | 'fail' | undefined from + * commit_review_lenses for that exact lens. + * + * Why this exists: statusOf/findNextOutcome is reviewer-level only. A reviewer can FAIL a ship on + * several distinct lenses at once; naively stamping one reviewer-level verdict onto every failing + * lens of that ship asserts THIS lens's problem was resolved when only the reviewer's overall + * next-attempt status is actually known — the other lens may be what actually got fixed. This + * scans for a 'pass' on the SAME lens first (real per-lens evidence); only when a candidate ship + * has no lens-level row at all for this lens does it fall back to the reviewer-level signal, and + * even then only treats a reviewer PASS or a clean ship (absent reviewer, exit_code 0) as fixed — + * a reviewer FAIL with no lens data keeps scanning rather than assuming this lens's fate either way. + */ +export function findNextLensOutcome( + shipsForBranch, + failShipId, + reviewer, + lens, + statusOf, + lensStatusOf, +) { + const chain = shipsForBranch ?? []; + const failIdx = chain.findIndex((s) => s?.ship_id === failShipId); + if (failIdx === -1) return { kind: 'no-fix-found', nextShipId: null }; + for (let i = failIdx + 1; i < chain.length; i += 1) { + const candidate = chain[i]; + const lensStatus = lensStatusOf(candidate.ship_id, reviewer, lens); + if (lensStatus === 'pass') return { kind: 'fixed', nextShipId: candidate.ship_id }; + if (lensStatus === 'fail') continue; // this specific lens still broken — keep scanning + // lensStatus undefined — no lens-level row for this lens on this candidate ship. Fall back + // to the reviewer-level signal, but only trust it in the directions that don't require + // assuming this lens's specific fate: + const status = statusOf(candidate.ship_id, reviewer); + if (status === 'pass') return { kind: 'fixed', nextShipId: candidate.ship_id }; + if (status === 'fail') continue; // reviewer still failing overall on SOME lens — could be a + // different lens than this one; not evidence this lens resolved, keep scanning. + if (candidate.exit_code === 0) return { kind: 'fixed', nextShipId: candidate.ship_id }; + // reviewer absent and not clean — no signal, keep scanning + } + return { kind: 'no-fix-found', nextShipId: null }; +} + +/** Decide which lens rows a reviewer-level FAIL mints fail→fix candidates for. + * + * Only a BLOCKING failed lens qualifies (allowlist — LensDisposition is 'blocking' | 'waived' | + * 'dropped_out_of_charter', items.mts): a waived lens is a human-labeled false positive that the + * waived-decoy loop owns (colliding here would silently win the merge), and a + * dropped_out_of_charter lens was dismissed by the gate's own cross-domain valve, so correlating + * a "fix" for either is meaningless. Null/absent disposition (pre-disposition-era rows) counts as + * blocking. + * + * Three cases, and the middle one is the trap — "no blocking lens failed" is NOT the same fact as + * "no lens breakdown exists", even though both leave the blocking filter empty: + * · breakdown with blocking failed lenses → those lenses, correlated per-lens + * · breakdown whose failed lenses are ALL waived/dropped → SKIP (`rows: []` + a skip reason). + * Falling through to the reviewer-level scan here would mint a gold from exactly the lenses + * the allowlist just excluded. + * · no lens breakdown recorded at all → `[null]`, the reviewer-level scan + * + * The caller counts `skipped` in its drop histogram rather than dropping the fail silently. */ +export function selectFailLensRows(lenses) { + const recorded = lenses ?? []; + const blocking = recorded.filter( + (l) => l.status === 'fail' && (l.disposition == null || l.disposition === 'blocking'), + ); + if (blocking.length > 0) return { rows: blocking, skipped: null }; + if (recorded.some((l) => l.status === 'fail')) + return { rows: [], skipped: 'all-failing-lenses-non-blocking' }; + return { rows: [null], skipped: null }; // null lens = no per-lens breakdown recorded +} + +// --------------------------------------------------------------------------------------------- +// diff_sha256 equality — guards against mislabeling an override-valve waiver (same diff bytes, +// reconciled to pass with no code change) as a "fix". That case belongs to the waived-decoy path +// (commit_review_lenses.disposition='waived'), not fail-fix. +// --------------------------------------------------------------------------------------------- + +export function isSameDiff(a, b) { + return typeof a === 'string' && typeof b === 'string' && a.length > 0 && a === b; +} + +/** + * True when there is SOME diff hash to reconstruct from — either side. False means neither the + * broken nor the fixed ship even has a commit_review_scope row for this reviewer (not just a + * missing archive), so the only evidence for the row is free-text failReason. commit_review_scope + * only starts covering ships from 2026-07-27 (see module docstring); most historical FAILs + * predate it, so a large share of fail-fix rows are expected to be `hasDiffEvidence: false`. + * Surfaced as its own field (rather than left implicit in diffSha256/nextDiffSha256 both being + * null) so downstream triage can threshold on it explicitly, the way propose.mts hard-drops + * empty-hunk bot candidates. + */ +export function hasDiffEvidence(diffSha256, nextDiffSha256) { + return Boolean(diffSha256) || Boolean(nextDiffSha256); +} + +// --------------------------------------------------------------------------------------------- +// Diff archive addressing. +// --------------------------------------------------------------------------------------------- + +/** `diffs/.diff.gz`, relative to the telemetry dir — mirrors diff-archive.mts exactly. */ +export function diffArchiveRelPath(diffSha256) { + if (!diffSha256) return null; + return `diffs/${diffSha256}.diff.gz`; +} + +/** + * Decides between inlining decompressed diff text and emitting a path ref, given the already + * gunzipped text (or null when the archive doesn't have this hash). Pure — the caller does the + * actual existsSync/gunzip IO and passes the result in. + */ +export function resolveDiffPayload(diffText, relPath) { + if (typeof diffText !== 'string') return { diffText: null, diffPath: null }; + if (Buffer.byteLength(diffText, 'utf8') <= INLINE_DIFF_CAP_BYTES) { + return { diffText, diffPath: null }; + } + return { diffText: null, diffPath: relPath ?? null }; +} + +// --------------------------------------------------------------------------------------------- +// Synthetic dedupe key / candidate row shaping. +// --------------------------------------------------------------------------------------------- + +/** `telemetry:////[/]` — the stable merge-by key (mirrors + * mine-bots' `url`). `kind` is part of the key so a fail-fix row and a decoy row derived from the + * same (ship, reviewer, lens) can never silently overwrite each other in mergeCandidates — the + * emit loops are mutually exclusive by disposition, and the key makes any future regression of + * that invariant visible in the output instead of a silent last-write-wins. */ +export function telemetryUrl(kind, shipId, reviewer, lens) { + const base = `telemetry://${kind}/${shipId}/${reviewer}`; + return lens ? `${base}/${lens}` : base; +} + +/** Best-effort single failure-reason string: prefer the lens's own issues_json, else the + * reviewer's overall commit_reviews.reason. Never throws on malformed issues_json. */ +export function pickFailReason(issuesJson, reviewReason) { + if (typeof issuesJson === 'string' && issuesJson.trim()) { + try { + const parsed = JSON.parse(issuesJson); + if (Array.isArray(parsed) && parsed.length > 0) { + // A non-empty array can still filter down to nothing (whitespace-only / non-string + // entries) — an empty join must fall through to reviewReason like every other + // no-usable-content path, not return '' and shadow the real reason at the `?? null` + // call sites (empty string survives ??). + const joined = parsed.filter((s) => typeof s === 'string' && s.trim()).join('\n'); + if (joined.trim()) return joined; + } + } catch { + // fall through to reviewReason + } + } + return typeof reviewReason === 'string' && reviewReason.trim() ? reviewReason : null; +} + +/** Builds one fail-fix candidate row. `diffPayload`/`nextDiffPayload` are the results of + * resolveDiffPayload (or `{diffText:null, diffPath:null}` when nothing was archived). */ +export function buildFailFixCandidate({ + shipId, + repo, + branch, + reviewer, + lens, + tsFail, + diffSha256, + bytesAvailable, + diffPayload, + failReason, + nextShipId, + nextDiffSha256, + nextBytesAvailable, + nextDiffPayload, + tsFix, +}) { + return { + kind: 'fail-fix', + url: telemetryUrl('fail-fix', shipId, reviewer, lens), + shipId, + repo, + branch, + reviewer, + lens: lens ?? null, + tsFail, + diffSha256: diffSha256 ?? null, + bytesAvailable: !!bytesAvailable, + diffText: diffPayload?.diffText ?? null, + diffPath: diffPayload?.diffPath ?? null, + failReason: failReason ?? null, + nextShipId, + nextDiffSha256: nextDiffSha256 ?? null, + nextBytesAvailable: !!nextBytesAvailable, + nextDiffText: nextDiffPayload?.diffText ?? null, + nextDiffPath: nextDiffPayload?.diffPath ?? null, + hasDiffEvidence: hasDiffEvidence(diffSha256, nextDiffSha256), + tsFix, + }; +} + +/** Builds one waived-decoy candidate row. `rationale` stays null until capture point 2 + * (`guard-review waive`) writes it into telemetry — never fabricated here. */ +export function buildWaivedCandidate({ + shipId, + repo, + branch, + reviewer, + lens, + tsFail, + diffSha256, + bytesAvailable, + diffPayload, + failReason, + disposition, + rationale = null, +}) { + return { + kind: 'waived-decoy', + url: telemetryUrl('waived-decoy', shipId, reviewer, lens), + shipId, + repo, + branch: branch ?? null, + reviewer, + lens: lens ?? null, + tsFail, + diffSha256: diffSha256 ?? null, + bytesAvailable: !!bytesAvailable, + diffText: diffPayload?.diffText ?? null, + diffPath: diffPayload?.diffPath ?? null, + failReason: failReason ?? null, + disposition, + rationale, + }; +} + +// --------------------------------------------------------------------------------------------- +// Merge (new wins by `url`, same contract as mine-bots.readExistingCandidates/merge). +// --------------------------------------------------------------------------------------------- + +export function mergeCandidates(existingByUrl, newRows) { + const merged = new Map(existingByUrl ?? new Map()); + for (const row of newRows ?? []) { + if (!row?.url) continue; + merged.set(row.url, row); + } + return merged; +} + +// --------------------------------------------------------------------------------------------- +// Histogram (kind × reviewer × bytesAvailable) for the stderr summary. +// --------------------------------------------------------------------------------------------- + +export function histogramKey(row) { + return `${row?.kind ?? 'unknown'} / ${row?.reviewer ?? 'unknown'} / bytesAvailable=${!!row?.bytesAvailable}`; +} + +export function buildHistogram(rows, keyFn = histogramKey) { + const counts = new Map(); + for (const row of rows ?? []) { + const k = keyFn(row); + counts.set(k, (counts.get(k) ?? 0) + 1); + } + return [...counts.entries()].sort((a, b) => b[1] - a[1]); +} diff --git a/gate-engine/review/eval/reviewers/mine-telemetry.mts b/gate-engine/review/eval/reviewers/mine-telemetry.mts new file mode 100644 index 0000000..dd0586b --- /dev/null +++ b/gate-engine/review/eval/reviewers/mine-telemetry.mts @@ -0,0 +1,391 @@ +#!/usr/bin/env node +// @ts-nocheck — BENCH-ONLY (excluded from tsc, see tsconfig.json exclude); loose types deliberate. + +/** + * mine-telemetry — turns real gate telemetry (`~/.claude-usage/usage.db`, the diff-bytes archive + * at `/diffs/.diff.gz`) into corpus-growth candidates, without any PR + * or bot involved. This is capture point 1 (fail→fix correlation) + the historical override-valve + * slice of capture point 2 (waived decoys), per docs/benchmarks/corpus-growth.md. + * + * bun mine-telemetry.mts [--repo ]... (default: devkit; or TELEMETRY_REPO_ALLOWLIST=a,b) + * + * Emits two candidate kinds, one row per (ship, reviewer[, lens]): + * - fail-fix — a reviewer FAILed ship S1 on a given lens; the next attempt on the same + * (repo, branch) where THAT SAME lens specifically passes (or, absent + * lens-level data, the reviewer overall passes / the ship ships clean with the + * reviewer silent) is the fix. Correlation is chronological within + * (repo, branch) and per-lens, not just per-reviewer — a reviewer failing on + * two distinct lenses in one ship gets two independently-scanned verdicts, not + * one reviewer-level verdict stamped onto both. See findNextLensOutcome (and + * findNextOutcome for the reviewer-level fallback used when a FAIL has no + * lens-level breakdown at all) in mine-telemetry-lib.mts for the exact scan + * rules. `hasDiffEvidence` flags rows where neither the FAIL nor the fix ship + * has a commit_review_scope row at all (common for FAILs predating + * 2026-07-27, when lens/scope capture began) — for those, reconstruction is + * failReason-only, not diffSha256-plus-failReason as it is for the rest. + * - waived-decoy — a `commit_review_lenses` row with disposition='waived' (the correctness + * override valve, overrides.mts). Rationale text is NOT in telemetry today + * (only `.devkit/correctness-overrides.json` on whichever checkout wrote it + * has it) — the field is always emitted as `null`, never fabricated. + * + * Repo scoping: `commit_ships.repo` spans every project ever shipped from on this machine, most + * of them unrelated private checkouts. Defaults to a `devkit`-only allowlist (see + * DEFAULT_REPO_ALLOWLIST below) so a bare `bun mine-telemetry.mts` never writes another project's + * diff text or finding prose into this repo's working tree; widen it explicitly via --repo/env + * when mining a different checkout on purpose. + * + * Read-only against the db (execFileSync sqlite3 -json, USAGE_DB env override) and the archive + * (existsSync + gunzipSync only) — this script writes nothing back to either. + * + * Output: raw/candidates-telemetry.jsonl (gitignored), atomic write (tmp+rename), merged by the + * synthetic key `telemetry:///[/]` (new wins), same contract as + * mine-bots.mts's `url`-keyed merge onto candidates.jsonl. + * + * Plan of record (scouted 2026-08-01, docs/benchmarks/corpus-growth.md): telemetry candidates do + * NOT flow through propose.mts. propose.mts's hard-drop `!originalCommitId` is unconditional and + * every telemetry candidate has none (a blocked ship never committed) — they would die before + * routing even ran, and its GitHub-contents-API enrichment step has no ref to fetch against + * anyway. Consumption is a SEPARATE later step: an adapt-stage agent reads + * raw/candidates-telemetry.jsonl directly (dedupe, drop `bytesAvailable:false` rows past some + * threshold, sort by recency/reviewer), never propose.mts. Do not wire this file into propose.mts. + */ + +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { gunzipSync } from 'node:zlib'; +import { + collectRepoArgs, + readCandidatesFile, + sqlite3Available, + sqliteJson, +} from './mine-common.mts'; +import { + buildFailFixCandidate, + buildHistogram, + buildWaivedCandidate, + diffArchiveRelPath, + findNextLensOutcome, + findNextOutcome, + groupShipsByRepoBranch, + hasDiffEvidence, + histogramKey, + isSameDiff, + mergeCandidates, + pickFailReason, + resolveDiffPayload, + selectFailLensRows, +} from './mine-telemetry-lib.mts'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const RAW_DIR = path.join(here, 'raw'); +const OUT = path.join(RAW_DIR, 'candidates-telemetry.jsonl'); + +// `commit_ships.repo` is the basename of whatever checkout shipped from — on a shared dev +// machine that spans every project the developer has ever shipped from, most of them private +// and unrelated to this one. Mirrors mine-bots.mts's `--repo owner/name` allowlist convention +// (default benord-labs/frink + norvalbv/devkit) so this miner never sweeps every repo in the +// collector db by default. Override with repeated `--repo ` args or a comma-separated +// TELEMETRY_REPO_ALLOWLIST env var. +const DEFAULT_REPO_ALLOWLIST = ['devkit']; + +function resolveRepoAllowlist() { + const repoArgs = collectRepoArgs(process.argv.slice(2)); + if (repoArgs.length > 0) return repoArgs; + const fromEnv = (process.env.TELEMETRY_REPO_ALLOWLIST || '') + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + return fromEnv.length > 0 ? fromEnv : DEFAULT_REPO_ALLOWLIST; +} + +// --------------------------------------------------------------------------------------------- +// sqlite3 access (read-only SELECTs only — mirrors mine-bots.mts's resolveScopeDb/sqliteJson). +// --------------------------------------------------------------------------------------------- + +function resolveDb() { + const dbPath = process.env.USAGE_DB || path.join(os.homedir(), '.claude-usage', 'usage.db'); + if (!existsSync(dbPath)) { + console.error(`mine-telemetry: no usage.db at ${dbPath} — nothing to mine, exiting`); + return null; + } + if (!sqlite3Available()) { + console.error('mine-telemetry: sqlite3 CLI not found on PATH — nothing to mine, exiting'); + return null; + } + return dbPath; +} + +// --------------------------------------------------------------------------------------------- +// Telemetry-dir resolution — mirrors judge/run-context.mts's telemetrySink() (dirname of the +// gate-events sink), replicated here rather than imported so this bench-only tool stays +// self-contained (same call as mine-bots.mts makes for the usage.db path). +// --------------------------------------------------------------------------------------------- + +function resolveTelemetryDir() { + const sink = + process.env.DEVKIT_GATE_EVENTS || + path.join(os.homedir(), '.devkit', 'telemetry', 'gate-events.jsonl'); + return path.dirname(sink); +} + +/** Reads + gunzips an archived diff if present. Never throws — a corrupt/missing archive entry + * degrades to `null`, matching diff-archive.mts's own fail-open contract. */ +function readArchivedDiff(telemetryDir, diffSha256) { + const rel = diffArchiveRelPath(diffSha256); + if (!rel) return { bytesAvailable: false, diffText: null, relPath: null }; + const abs = path.join(telemetryDir, rel); + if (!existsSync(abs)) return { bytesAvailable: false, diffText: null, relPath: rel }; + try { + return { + bytesAvailable: true, + diffText: gunzipSync(readFileSync(abs)).toString('utf8'), + relPath: rel, + }; + } catch (e) { + console.error(`mine-telemetry: archive read failed for ${rel} (${e.message?.split('\n')[0]})`); + return { bytesAvailable: false, diffText: null, relPath: rel }; + } +} + +// --------------------------------------------------------------------------------------------- +// Merge / atomic write (mirrors mine-bots.mts's readExistingCandidates + tmp+rename write). +// --------------------------------------------------------------------------------------------- + +function writeAtomic(rows) { + mkdirSync(RAW_DIR, { recursive: true }); + const tmp = `${OUT}.tmp`; + writeFileSync(tmp, `${rows.map((r) => JSON.stringify(r)).join('\n')}\n`); + renameSync(tmp, OUT); +} + +// --------------------------------------------------------------------------------------------- +// Main. +// --------------------------------------------------------------------------------------------- + +function main() { + const dbPath = resolveDb(); + if (!dbPath) { + process.exit(0); + } + + let ships = []; + let reviews = []; + let scopeRows = []; + let lensRows = []; + try { + ships = sqliteJson( + dbPath, + 'SELECT ship_id, repo, branch, ts_start, exit_code FROM commit_ships;', + ); + reviews = sqliteJson(dbPath, 'SELECT ship_id, reviewer, status, reason FROM commit_reviews;'); + scopeRows = sqliteJson( + dbPath, + 'SELECT ship_id, reviewer, diff_sha256 FROM commit_review_scope;', + ); + lensRows = sqliteJson( + dbPath, + 'SELECT ship_id, reviewer, lens, status, disposition, issues_json FROM commit_review_lenses;', + ); + } catch (e) { + console.error(`mine-telemetry: query failed (${e.message?.split('\n')[0]}) — exiting`); + process.exit(0); + } + + const repoAllowlist = new Set(resolveRepoAllowlist()); + const shipsBeforeAllowlist = ships.length; + ships = ships.filter((s) => repoAllowlist.has(s.repo)); + const allowedShipIds = new Set(ships.map((s) => s.ship_id)); + reviews = reviews.filter((r) => allowedShipIds.has(r.ship_id)); + scopeRows = scopeRows.filter((s) => allowedShipIds.has(s.ship_id)); + lensRows = lensRows.filter((l) => allowedShipIds.has(l.ship_id)); + const excludedByRepo = shipsBeforeAllowlist - ships.length; + + const telemetryDir = resolveTelemetryDir(); + if (!existsSync(telemetryDir)) { + console.error( + `mine-telemetry: no telemetry dir at ${telemetryDir} — bytesAvailable will be false everywhere`, + ); + } + + const shipsById = new Map(ships.map((s) => [s.ship_id, s])); + const { groups, indexByShipId, skippedNullBranch } = groupShipsByRepoBranch(ships); + + const reviewStatus = new Map(); // `${shipId}::${reviewer}` -> status + for (const r of reviews) { + reviewStatus.set(`${r.ship_id}::${r.reviewer}`, r.status); + } + const statusOf = (shipId, reviewer) => reviewStatus.get(`${shipId}::${reviewer}`); + + const scopeByShipReviewer = new Map(); // same key -> diff_sha256 + for (const s of scopeRows) + scopeByShipReviewer.set(`${s.ship_id}::${s.reviewer}`, s.diff_sha256 ?? null); + + const lensesByShipReviewer = new Map(); // key -> LensRow[] + const lensStatus = new Map(); // `${shipId}::${reviewer}::${lens}` -> status + const waivedLenses = []; + for (const l of lensRows) { + const key = `${l.ship_id}::${l.reviewer}`; + if (!lensesByShipReviewer.has(key)) lensesByShipReviewer.set(key, []); + lensesByShipReviewer.get(key).push(l); + lensStatus.set(`${l.ship_id}::${l.reviewer}::${l.lens}`, l.status); + if (l.disposition === 'waived') waivedLenses.push(l); + } + const lensStatusOf = (shipId, reviewer, lens) => + lensStatus.get(`${shipId}::${reviewer}::${lens}`); + + const newRows = []; + const dropReasons = {}; + const bumpDrop = (reason) => { + dropReasons[reason] = (dropReasons[reason] ?? 0) + 1; + }; + + // ---- fail-fix ------------------------------------------------------------------------------- + const failedReviews = reviews.filter((r) => r.status === 'fail'); + for (const fr of failedReviews) { + const ship = shipsById.get(fr.ship_id); + if (!ship) { + bumpDrop('fail-fix:ship-missing'); + continue; + } + const loc = indexByShipId.get(fr.ship_id); + if (!loc) { + // null/blank branch — uncorrelatable, already counted in skippedNullBranch. + continue; + } + const chain = groups.get(loc.key); + // diffSha256 (the broken side) is per (ship, reviewer), not per lens — same for every lens + // of this fail, so resolve it and its archive once, outside the per-lens loop below. + const diffSha256 = scopeByShipReviewer.get(`${fr.ship_id}::${fr.reviewer}`) ?? null; + const archived = readArchivedDiff(telemetryDir, diffSha256); + + // Blocking-only allowlist, and the "all failing lenses were waived/dropped" case skips this + // fail outright instead of falling back to a reviewer-level candidate — see + // selectFailLensRows in mine-telemetry-lib.mts for the full rule. + const { rows, skipped } = selectFailLensRows( + lensesByShipReviewer.get(`${fr.ship_id}::${fr.reviewer}`), + ); + if (skipped) { + bumpDrop(`fail-fix:${skipped}`); + continue; + } + + for (const lensRow of rows) { + const lens = lensRow?.lens ?? null; + // Per-lens correlation when this FAIL has lens-level data: each failing lens of a + // multi-lens FAIL is independently scanned for ITS OWN pass, not stamped with one shared + // reviewer-level verdict (see findNextLensOutcome's docstring in mine-telemetry-lib.mts). + // Falls back to the reviewer-level scan only when there's no lens breakdown at all. + const { kind, nextShipId } = + lens !== null + ? findNextLensOutcome(chain, fr.ship_id, fr.reviewer, lens, statusOf, lensStatusOf) + : findNextOutcome(chain, fr.ship_id, fr.reviewer, statusOf); + if (kind !== 'fixed') { + bumpDrop(`fail-fix:${kind}`); + continue; + } + const nextDiffSha256 = scopeByShipReviewer.get(`${nextShipId}::${fr.reviewer}`) ?? null; + if (isSameDiff(diffSha256, nextDiffSha256)) { + // Same bytes reconciled to pass with no code change — an override-valve waiver, not a + // fix. That case is captured separately via commit_review_lenses.disposition='waived'. + bumpDrop('fail-fix:same-diff-override-not-fix'); + continue; + } + const nextShip = shipsById.get(nextShipId); + const nextArchived = readArchivedDiff(telemetryDir, nextDiffSha256); + newRows.push( + buildFailFixCandidate({ + shipId: fr.ship_id, + repo: ship.repo, + branch: ship.branch, + reviewer: fr.reviewer, + lens, + tsFail: ship.ts_start, + diffSha256, + bytesAvailable: archived.bytesAvailable, + diffPayload: resolveDiffPayload(archived.diffText, archived.relPath), + failReason: pickFailReason(lensRow?.issues_json, fr.reason), + nextShipId, + nextDiffSha256, + nextBytesAvailable: nextArchived.bytesAvailable, + nextDiffPayload: resolveDiffPayload(nextArchived.diffText, nextArchived.relPath), + tsFix: nextShip?.ts_start ?? null, + }), + ); + } + } + + // ---- waived-decoy ----------------------------------------------------------------------------- + for (const l of waivedLenses) { + const ship = shipsById.get(l.ship_id); + if (!ship) { + bumpDrop('waived-decoy:ship-missing'); + continue; + } + const diffSha256 = scopeByShipReviewer.get(`${l.ship_id}::${l.reviewer}`) ?? null; + const archived = readArchivedDiff(telemetryDir, diffSha256); + newRows.push( + buildWaivedCandidate({ + shipId: l.ship_id, + repo: ship.repo, + branch: ship.branch ?? null, + reviewer: l.reviewer, + lens: l.lens, + tsFail: ship.ts_start, + diffSha256, + bytesAvailable: archived.bytesAvailable, + diffPayload: resolveDiffPayload(archived.diffText, archived.relPath), + failReason: pickFailReason(l.issues_json, null), + disposition: l.disposition, + // Rationale is NOT captured in telemetry today (only in whichever checkout's + // .devkit/correctness-overrides.json wrote it) — never fabricated here. + rationale: null, + }), + ); + } + + // ---- merge + write ------------------------------------------------------------------------ + // Prune any existing rows outside the CURRENT repo allowlist before merging in new ones. Without + // this, a prior run made with a wider (or no) allowlist leaves other-repo rows — including their + // full diff text — sitting in this file forever, since mergeCandidates only ever adds/updates by + // url and never removes. The allowlist must be enforced on every write, not just on what a given + // run discovers fresh. + const existing = readCandidatesFile(OUT); + let prunedByRepo = 0; + for (const [url, row] of existing) { + if (!repoAllowlist.has(row?.repo)) { + existing.delete(url); + prunedByRepo += 1; + } + } + const merged = mergeCandidates(existing, newRows); + const rows = [...merged.values()]; + writeAtomic(rows); + + // ---- stderr summary ------------------------------------------------------------------------- + const hist = buildHistogram(rows, histogramKey); + const hollowFailFix = rows.filter( + (r) => r.kind === 'fail-fix' && hasDiffEvidence(r.diffSha256, r.nextDiffSha256) === false, + ).length; + console.error( + [ + `mine-telemetry: ${rows.length} candidates → ${path.relative(here, OUT)} (${newRows.length} new/updated this run)`, + ` repo allowlist: ${[...repoAllowlist].join(', ')} (excluded ${excludedByRepo} ships from other repos, pruned ${prunedByRepo} stale off-allowlist rows from the output file)`, + ` skipped (null/blank branch): ${skippedNullBranch}`, + ` fail-fix rows with NO diff evidence at all (diffSha256 and nextDiffSha256 both null — failReason-only): ${hollowFailFix}`, + ` drops: ${ + Object.entries(dropReasons) + .sort((a, b) => b[1] - a[1]) + .map(([k, v]) => `${k}:${v}`) + .join(', ') || '—' + }`, + ' by kind × reviewer × bytesAvailable:', + ...hist.map(([k, v]) => ` ${k}: ${v}`), + ].join('\n'), + ); +} + +const invokedDirectly = + process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) main();