chore(test): node-suite differential runner + regression guard

scripts/node_suite_regression_check.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Regression guard for the print-and-diff node-suite.
+
+Runs `scripts/node_suite_run.py` (pre-warm + fast/slow lanes) and compares the
+per-module pass counts against a committed floor baseline. FAILS (exit 1) if any
+baselined module drops below its floor. Improvements are always accepted and are
+reported as `+N` so the baseline can be ratcheted up over time.
+
+This exists because the node-suite is NOT part of the per-PR CI gate (the parity
+job is opt-in and runs node 22, while the real oracle is node 26), so a module
+can silently regress and still merge green — which is exactly how node:dns once
+went 83% -> 0% behind a green build. Run this in the node-26 environment (the box)
+on a schedule, or before cutting a release.
+
+Usage:
+  node_suite_regression_check.py <perry-bin> <repo-root> [baseline.json]
+
+Exit codes: 0 = no regressions (improvements ok), 1 = at least one regression,
+            2 = harness error (could not run / parse).
+"""
+import json
+import os
+import re
+import subprocess
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+REPO_DEFAULT = os.path.dirname(HERE)
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(__doc__)
+        return 2
+    perry, root = sys.argv[1], sys.argv[2]
+    baseline_path = sys.argv[3] if len(sys.argv) > 3 else os.path.join(
+        root, "test-parity", "node_suite_baseline.json")
+
+    if not os.path.exists(baseline_path):
+        print(f"ERROR: baseline not found: {baseline_path}", file=sys.stderr)
+        return 2
+    baseline = json.load(open(baseline_path)).get("modules", {})
+
+    runner = os.path.join(root, "scripts", "node_suite_run.py")
+    proc = subprocess.run([sys.executable, runner, perry, root],
+                          capture_output=True, text=True)
+    sys.stderr.write(proc.stderr)
+    print(proc.stdout)
+    # Fail closed: any non-zero runner exit means we cannot trust the table
+    # (crash/timeout could leave partial output), so don't risk parsing it.
+    if proc.returncode != 0:
+        print(f"ERROR: runner exited {proc.returncode}", file=sys.stderr)
+        return 2
+
+    # Parse "module  pass  total  %" rows from the runner table.
+    # The header row ("module  pass  total  %") can't match because pass/total
+    # are not digits, so no name-based exclusion is needed — and excluding the
+    # name "module" would wrongly drop the real node:module module.
+    current = {}
+    for line in proc.stdout.splitlines():
+        m = re.match(r"^(\S+)\s+(\d+)\s+(\d+)\s+[\d.]+", line)
+        if m:
+            current[m.group(1)] = {"pass": int(m.group(2)), "total": int(m.group(3))}
+
+    regressions, improvements = [], []
+    for mod, floor in baseline.items():
+        cur = current.get(mod)
+        if cur is None:
+            regressions.append(f"{mod}: MISSING from run (was {floor['pass']}/{floor['total']})")
+            continue
+        if cur["pass"] < floor["pass"]:
+            regressions.append(
+                f"{mod}: {cur['pass']}/{cur['total']} < floor {floor['pass']}/{floor['total']}  (-{floor['pass'] - cur['pass']})")
+        elif cur["pass"] > floor["pass"]:
+            improvements.append(f"{mod}: {cur['pass']}/{cur['total']}  (+{cur['pass'] - floor['pass']})")
+
+    print("\n=== node-suite regression check ===")
+    if improvements:
+        print("improvements (ratchet the baseline up):")
+        for s in improvements:
+            print("  + " + s)
+    if regressions:
+        print("REGRESSIONS:")
+        for s in regressions:
+            print("  ! " + s)
+        return 1
+    print("OK — no module dropped below its floor.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/node_suite_run.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Differential runner for the print-and-diff node-suite (test-parity/node-suite).
+
+For every `test-parity/node-suite/<module>/**/*.ts`, run `node <t>` and
+`perry <t> -o out && out`, then compare stdout (trailing whitespace ignored).
+Prints a per-module pass/total table plus an overall figure.
+
+Two correctness measures learned the hard way (see CHANGELOG / project memory):
+
+1. Pre-warm pass — compile one test per module SERIALLY first so each module's
+   auto-optimize runtime/stdlib cache (e.g. the crypto feature) is built before
+   the timed run. Otherwise the first test of a crypto-feature module eats a
+   multi-minute cold rebuild that blows the per-test compile timeout and the
+   whole module is scored as `perry_err` (this once made dns look 0% and http
+   47% when both are actually 100%).
+
+2. Low-concurrency lane — server/timing modules bind ports, spawn processes, or
+   assert on event-loop/timer ordering. Under the wide parallel pool they suffer
+   port contention and timing races, producing false `perry_err`/`diff`. They
+   run STRICTLY SEQUENTIALLY so their numbers are trustworthy; everything else
+   stays parallel.
+
+Usage: node_suite_run.py <perry-bin> <repo-root> [comma-separated-modules]
+"""
+import os, subprocess, sys, tempfile
+from concurrent.futures import ThreadPoolExecutor
+from collections import defaultdict
+
+PERRY = sys.argv[1]
+ROOT = sys.argv[2]
+MODS = sys.argv[3].split(",") if len(sys.argv) > 3 and sys.argv[3] else None
+NODE = os.environ.get("NODE_BIN", "node")
+
+# Modules that must run one-at-a-time (port binding / process spawn / event-loop
+# or timer ordering). Parallelism corrupts their results.
+SLOW_MODULES = {
+    "http", "http2", "https", "net", "dgram", "tls", "cluster", "dns",
+    "stream", "child_process", "worker_threads", "inspector",
+    "inspector-promises", "repl", "diagnostics_channel", "timers", "fetch",
+}
+
+tests = []
+base = os.path.join(ROOT, "test-parity", "node-suite")
+for mod in (MODS or sorted(os.listdir(base))):
+    md = os.path.join(base, mod)
+    if not os.path.isdir(md):
+        continue
+    for dp, _, files in os.walk(md):
+        for f in files:
+            if f.endswith(".ts") and not f.endswith(".d.ts"):
+                tests.append((mod, os.path.join(dp, f)))
+
+
+def run_one(args):
+    mod, path = args
+    try:
+        n = subprocess.run([NODE, path], capture_output=True, text=True, timeout=30)
+    except Exception:
+        return (mod, "node_err")
+    # A non-zero node exit can be intentional (the test exercises an error path),
+    # so we don't bucket it as node_err; we require Perry to match BOTH stdout and
+    # the exit code below, which keeps genuine error-path parity counted as pass.
+    with tempfile.TemporaryDirectory() as td:
+        out = os.path.join(td, "o")
+        try:
+            c = subprocess.run([PERRY, path, "-o", out], capture_output=True, text=True, timeout=120)
+            if c.returncode != 0:
+                return (mod, "compile_fail")
+            p = subprocess.run([out], capture_output=True, text=True, timeout=30)
+        except Exception:
+            return (mod, "perry_err")
+    # Match stdout byte-for-byte (ignore only trailing-newline noise, not leading
+    # whitespace) AND exit code — so a Perry crash that happened to print matching
+    # output before dying is a diff, not a false pass.
+    ok = (n.stdout.rstrip("\n") == p.stdout.rstrip("\n")) and (n.returncode == p.returncode)
+    return (mod, "pass" if ok else "diff")
+
+
+# --- pre-warm one test per module serially ---
+seen = set()
+warm = [t for t in tests if t[0] not in seen and not seen.add(t[0])]
+sys.stderr.write(f"pre-warming auto-opt cache for {len(warm)} module(s)...\n")
+sys.stderr.flush()
+for mod, path in warm:
+    with tempfile.TemporaryDirectory() as td:
+        try:
+            subprocess.run([PERRY, path, "-o", os.path.join(td, "o")], capture_output=True, text=True, timeout=600)
+        except Exception:
+            pass
+sys.stderr.write("pre-warm done\n")
+sys.stderr.flush()
+
+# --- fast lane (parallel) + slow lane (sequential) ---
+fast = [t for t in tests if t[0] not in SLOW_MODULES]
+slow = [t for t in tests if t[0] in SLOW_MODULES]
+res = defaultdict(lambda: defaultdict(int))
+sys.stderr.write(f"fast lane: {len(fast)} tests @6, slow lane: {len(slow)} tests @1\n")
+sys.stderr.flush()
+with ThreadPoolExecutor(max_workers=6) as ex:
+    for mod, outcome in ex.map(run_one, fast):
+        res[mod][outcome] += 1
+for t in slow:
+    mod, outcome = run_one(t)
+    res[mod][outcome] += 1
+
+# --- report ---
+tot_p = tot = 0
+print("%-20s %6s %6s  %5s" % ("module", "pass", "total", "%"))
+for mod in sorted(res):
+    a = res[mod]
+    p = a.get("pass", 0)
+    t = sum(a.values())
+    tot_p += p
+    tot += t
+    extra = " ".join(f"{k}={v}" for k, v in a.items() if k != "pass" and v)
+    print("%-20s %6d %6d  %5.1f  %s" % (mod, p, t, 100 * p / t if t else 0, extra))
+print("-" * 44)
+print("OVERALL node-suite: %d/%d (%.1f%%)" % (tot_p, tot, 100 * tot_p / tot if tot else 0))

test-parity/node_suite_baseline.json

@@ -0,0 +1,63 @@
+{
+  "_schema": {
+    "description": "Floor baseline for scripts/node_suite_regression_check.py. Each module's run must produce pass >= floor.pass; dropping below is a regression (exit 1). Improvements are always accepted and reported as ratchet candidates. Captured in the node-26 environment with scripts/node_suite_run.py (pre-warm + fast/slow lanes).",
+    "oracle": "node v26.3.0 on Linux (the box)",
+    "note": "Deterministic modules are floored at full pass. Timing/racy modules (http2, net, stream, diagnostics_channel, fs-promises) carry a small margin below observed pass so ordinary flake does not false-alarm; the guard still catches real regressions, which are large (e.g. dns 6->0, http 19->9)."
+  },
+  "overall": { "pass": 2781, "total": 2863, "pct": 97.1 },
+  "modules": {
+    "assert": { "pass": 70, "total": 70 },
+    "async_hooks": { "pass": 5, "total": 5 },
+    "bigint": { "pass": 3, "total": 3 },
+    "buffer": { "pass": 134, "total": 134 },
+    "child_process": { "pass": 26, "total": 26 },
+    "cluster": { "pass": 1, "total": 1 },
+    "console": { "pass": 108, "total": 119 },
+    "constants": { "pass": 4, "total": 4 },
+    "crypto": { "pass": 240, "total": 242 },
+    "dgram": { "pass": 4, "total": 4 },
+    "diagnostics_channel": { "pass": 65, "total": 69 },
+    "dns": { "pass": 6, "total": 6 },
+    "domain": { "pass": 3, "total": 3 },
+    "events": { "pass": 65, "total": 69 },
+    "fetch": { "pass": 12, "total": 12 },
+    "fs": { "pass": 168, "total": 175 },
+    "fs-promises": { "pass": 76, "total": 82 },
+    "globals": { "pass": 107, "total": 115 },
+    "http": { "pass": 19, "total": 19 },
+    "http2": { "pass": 8, "total": 9 },
+    "https": { "pass": 5, "total": 5 },
+    "inspector": { "pass": 2, "total": 3 },
+    "inspector-promises": { "pass": 1, "total": 1 },
+    "module": { "pass": 28, "total": 28 },
+    "net": { "pass": 12, "total": 13 },
+    "node-core": { "pass": 2, "total": 2 },
+    "object": { "pass": 22, "total": 23 },
+    "os": { "pass": 39, "total": 39 },
+    "path": { "pass": 92, "total": 92 },
+    "perf_hooks": { "pass": 83, "total": 83 },
+    "process": { "pass": 95, "total": 96 },
+    "punycode": { "pass": 8, "total": 8 },
+    "querystring": { "pass": 59, "total": 59 },
+    "readline": { "pass": 12, "total": 12 },
+    "repl": { "pass": 4, "total": 4 },
+    "sea": { "pass": 1, "total": 1 },
+    "sqlite": { "pass": 1, "total": 1 },
+    "stream": { "pass": 760, "total": 801 },
+    "string": { "pass": 2, "total": 2 },
+    "string_decoder": { "pass": 36, "total": 36 },
+    "sys": { "pass": 2, "total": 2 },
+    "test": { "pass": 7, "total": 11 },
+    "timers": { "pass": 90, "total": 90 },
+    "tls": { "pass": 3, "total": 3 },
+    "trace_events": { "pass": 6, "total": 6 },
+    "tty": { "pass": 31, "total": 32 },
+    "url": { "pass": 67, "total": 67 },
+    "util": { "pass": 84, "total": 86 },
+    "v8": { "pass": 4, "total": 4 },
+    "vm": { "pass": 8, "total": 8 },
+    "wasi": { "pass": 3, "total": 3 },
+    "worker_threads": { "pass": 17, "total": 17 },
+    "zlib": { "pass": 58, "total": 58 }
+  }
+}