From edf66f48041216ae3ebdbac7c1539e01e7a243cc Mon Sep 17 00:00:00 2001 From: norvalbv Date: Sun, 2 Aug 2026 09:33:25 +0100 Subject: [PATCH] =?UTF-8?q?bench(reviewer-eval):=20waive=20hunk=20drops=20?= =?UTF-8?q?for=20rebutted=20candidates=20=E2=80=94=20decoys=20don't=20need?= =?UTF-8?q?=20the=20hunk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebutted bot threads are the scarce mined-decoy source, and propose.mts's empty-hunk / hunk-too-long drops were killing 5 of them. A decoy fixture is built from baseFileContent (contents API at originalCommitId) plus the bot's rebutted claim — the diff hunk locates a defect being FIXED, which a decoy by definition doesn't have. Golds (outcome=fixed) keep both hunk checks. Re-derived from candidates.jsonl (749 rows, 34 rebutted): 25 already survived, 4 died hunk-too-long, 1 empty-hunk; the handover's "mostly fail hard drops" premise was wrong. Verified live: correctness routing 173 -> 176 (+3 in-charter decoys); hunk-too-long 113 -> 109; empty-hunk drop gone; no-line untouched (zero rebutted rows die there, and a row with neither line nor hunk gives the adapter nothing to anchor). Co-Authored-By: Claude Fable 5 --- gate-engine/review/eval/reviewers/propose.mts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gate-engine/review/eval/reviewers/propose.mts b/gate-engine/review/eval/reviewers/propose.mts index 21c982a..01a80e0 100644 --- a/gate-engine/review/eval/reviewers/propose.mts +++ b/gate-engine/review/eval/reviewers/propose.mts @@ -74,6 +74,10 @@ function hardDropReason(c) { if (!c.originalCommitId) return 'missing-original-commit'; if (c.line === null && c.originalLine === null) return 'no-line'; if (c.outcomeEvidence === 'outdated-only') return 'outdated-only'; + // Rebutted candidates mint DECOYS — the scarce corpus resource. Their fixture is built from + // baseFileContent + the bot's (rebutted) claim, not from the diff hunk, so hunk absence/size + // must not drop them. Golds (fixed) still need the hunk to locate the defect being fixed. + if (c.outcome === 'rebutted') return null; if (!c.diffHunk || c.diffHunk.length === 0) return 'empty-hunk'; if ((c.hunkLen ?? c.diffHunk.length) > MAX_HUNK_LEN) return 'hunk-too-long'; return null;