#!/usr/bin/env node /** * Evidence gate: mechanically checks that an AI agent's *claimed* evidence is * real, rather than trusting the transcript. Two modes: * * --citations Resolve `file:line` / `file:start-end` citations * in an evidence artifact; fail if a path is * missing, absolute/outside the repo, or the line * is out of range. * --tripwires Scan `git diff HEAD` for fake-green tampering: * deleted test files, newly skipped tests, and new * TODO/FIXME left in changed code. * * With no mode flag, runs --tripwires (citations need an artifact argument). * * This is a GATE, not a convenience hook: it fails loud. * exit 0 = clean exit 1 = violations found exit 2 = usage / internal error * (Exit 2 is kept distinct from 1 so a crash can never masquerade as a * "violation", per scripts/test-verify-evidence.cjs.) * * Precision-first by design: it would rather miss an oddly-shaped citation or an * unusually-named test than cry wolf — a gate that cries wolf gets disabled. It * deliberately does NOT attempt deep test-tamper analysis (assertion-count * deltas, mutation proofs); that is out of scope for this slice. * * Zero dependencies. Usage: node scripts/verify-evidence.cjs [--citations ] [--tripwires] */ "use strict"; const fs = require("fs"); const path = require("path"); const { spawnSync } = require("child_process"); const USAGE = "Usage: node scripts/verify-evidence.cjs [--citations ] [--tripwires]"; // ---------------------------------------------------------------- citations // A citation is `path:line` or `path:start-end`, backtick-optional. To avoid // false positives from prose ("ratio 3:2", "12:30") and URLs, the path token // must contain a "." or "/", URLs are stripped first, and a match preceded by a // word char or ":" is rejected. const CITATION_RE = /(? lineCount) { violations.push( `${c.raw} — line out of range (file has ${lineCount} lines)` ); } } return { violations, checked: citations.length }; } // ---------------------------------------------------------------- tripwires function isTestFile(p) { const f = p.toLowerCase(); return ( /(^|\/)tests?\//.test(f) || /(^|\/)__tests__\//.test(f) || /\.(test|spec)\.[a-z0-9]+$/.test(f) || /(^|\/)test_[^/]*\.py$/.test(f) || /_test\.py$/.test(f) || // "test"/"spec" as a separator-bounded token in the basename — catches // e.g. app.test.js, foo_spec.rb, but not latest.js / inspector.js. /(^|[\/._-])(test|spec)([\/._-]|$)/.test(f) ); } const SKIP_PATTERNS = [ { name: "skipped test (.skip)", re: /\.skip\s*\(/ }, { name: "skipped test (xit/xdescribe/xtest)", re: /\bx(it|describe|test)\s*\(/ }, // Matches @skip, @pytest.mark.skip, @pytest.mark.skipif, @unittest.skip, // @unittest.skipUnless, @unittest.skipIf. { name: "skipped test (@skip decorator)", re: /@(?:pytest\.mark\.|unittest\.)?skip/ }, ]; const MARKER_RE = /\b(TODO|FIXME)\b/; function getDiff() { const override = process.env.VERIFY_EVIDENCE_DIFF_FILE; if (override) { try { return { text: fs.readFileSync(override, "utf8") }; } catch { return { unavailable: `cannot read VERIFY_EVIDENCE_DIFF_FILE: ${override}` }; } } const r = spawnSync("git", ["diff", "HEAD"], { encoding: "utf8", timeout: 10000 }); if (r.error || r.status !== 0) { return { unavailable: "git diff HEAD unavailable (not a git repo?)" }; } return { text: r.stdout || "" }; } function checkTripwires() { const diff = getDiff(); if (diff.unavailable) return { skipped: diff.unavailable }; const violations = []; const lines = diff.text.split("\n"); let aPath = null; let bPath = null; let curFile = null; let newLineNo = 0; let renameFrom = null; const stripPrefix = (s) => s === "/dev/null" ? "/dev/null" : s.replace(/^[ab]\//, ""); for (const line of lines) { // A new file section resets all per-file state, so location and deletion // attribution never leak across files in a multi-file diff. if (line.startsWith("diff --git ")) { aPath = null; bPath = null; curFile = null; newLineNo = 0; renameFrom = null; continue; } if (line.startsWith("rename from ")) { renameFrom = line.slice("rename from ".length).trim(); continue; } if (line.startsWith("rename to ")) { const renameTo = line.slice("rename to ".length).trim(); // Renaming a test file to a non-test path removes it from the suite. if (renameFrom && isTestFile(renameFrom) && !isTestFile(renameTo)) { violations.push(`test file renamed out of the suite: ${renameFrom} → ${renameTo}`); } continue; } if (line.startsWith("--- ")) { aPath = stripPrefix(line.slice(4).trim()); continue; } if (line.startsWith("+++ ")) { bPath = stripPrefix(line.slice(4).trim()); curFile = bPath === "/dev/null" ? aPath : bPath; // A file whose new side is /dev/null was deleted. if (bPath === "/dev/null" && aPath && isTestFile(aPath)) { violations.push(`deleted test file: ${aPath}`); } continue; } const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line); if (hunk) { newLineNo = parseInt(hunk[1], 10); continue; } // Added content line (not the "+++ " file header, handled above). if (line.startsWith("+")) { const content = line.slice(1); const loc = curFile ? `${curFile}:${newLineNo}` : "(unknown)"; for (const s of SKIP_PATTERNS) { if (s.re.test(content)) violations.push(`${s.name} added at ${loc}`); } if (MARKER_RE.test(content)) { violations.push(`TODO/FIXME added at ${loc}: ${content.trim()}`); } newLineNo++; continue; } if (line.startsWith(" ")) { newLineNo++; continue; } // "-" lines and metadata: no new-side advance } return { violations }; } // ---------------------------------------------------------------- main function main() { const argv = process.argv.slice(2); if (argv.includes("--help") || argv.includes("-h")) { console.log(USAGE); return 0; } let artifact = null; let doCitations = false; let doTripwires = false; for (let i = 0; i < argv.length; i++) { if (argv[i] === "--citations") { doCitations = true; artifact = argv[i + 1] && !argv[i + 1].startsWith("-") ? argv[++i] : null; } else if (argv[i] === "--tripwires") { doTripwires = true; } else { console.error(`Unknown argument: ${argv[i]}\n${USAGE}`); return 2; } } // Default: no mode flag → tripwires (citations require an artifact arg). if (!doCitations && !doTripwires) doTripwires = true; const findings = []; let anyViolation = false; if (doCitations) { if (!artifact) { console.error(`--citations requires an artifact path\n${USAGE}`); return 2; } const res = checkCitations(artifact); if (res.usageError) { console.error(res.usageError); return 2; } if (res.violations.length) { anyViolation = true; findings.push( `citations: ${res.violations.length} unresolved of ${res.checked} checked` ); for (const v of res.violations) findings.push(` ✗ ${v}`); } else { findings.push(`citations: ${res.checked} checked, all resolve`); } } if (doTripwires) { const res = checkTripwires(); if (res.skipped) { findings.push(`tripwires: skipped — ${res.skipped}`); } else if (res.violations.length) { anyViolation = true; findings.push(`tripwires: ${res.violations.length} found`); for (const v of res.violations) findings.push(` ✗ ${v}`); } else { findings.push(`tripwires: none`); } } const out = findings.join("\n"); if (anyViolation) { console.error(out); console.error("\nFAIL — evidence gate found unverifiable claims."); return 1; } console.log(out); console.log("\nOK — evidence gate passed."); return 0; } try { process.exit(main()); } catch (err) { // Internal error is exit 2, never 1 — a crash must not read as a violation. console.error(`verify-evidence: internal error: ${err && err.message}`); process.exit(2); }