diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 161e0ce..06ee473 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -14,3 +14,5 @@ jobs: run: node scripts/validate-plugin.cjs - name: Check frontmatter token budgets run: node scripts/token-report.cjs --check + - name: Test the evidence gate + run: node scripts/test-verify-evidence.cjs diff --git a/AGENTS.md b/AGENTS.md index ceb5b33..b53eb8a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,9 @@ docs website (`website/`, Astro Starlight). The only repo-level check is - Plugin: nothing to build. Lint before committing: `node scripts/validate-plugin.cjs` — validates every SKILL.md against the 8-section anatomy and plugin/marketplace version sync; `node scripts/token-report.cjs --check` — enforces token budgets - on skill/agent frontmatter (both run in CI, `.github/workflows/validate.yml`). + on skill/agent frontmatter; `node scripts/test-verify-evidence.cjs` — exercises + the evidence gate (`scripts/verify-evidence.cjs`: citation resolution + + fake-green tripwires). All three run in CI, `.github/workflows/validate.yml`. - Test by installing locally: `/plugin marketplace add ` → `/plugin install claudekit` → `/claudekit:init` - Website (`cd website/`): `npm install`, `npm run dev`, `npm run build`, diff --git a/CHANGELOG.md b/CHANGELOG.md index 2698575..5820faf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 skill/agent frontmatter with named budgets, enforced in CI via `--check` (#2) - CI workflow `.github/workflows/validate.yml` running the anatomy and token-budget checks on pushes to `main` and PRs (#1, #2) +- `scripts/verify-evidence.cjs` — evidence gate that mechanically checks an + agent's claimed evidence: resolves `file:line` citations against real + paths/ranges (`--citations`) and scans `git diff HEAD` for fake-green + tampering — deleted test files, added test skips, new TODO/FIXME + (`--tripwires`); loud gate (exit 1 violations, 2 usage/crash). Deliberately + does not attempt deep test-tamper analysis +- `scripts/verify-evidence-hook.cjs` — fail-open Stop/PostToolUse wrapper that + surfaces tripwire findings as advisory, never blocking +- `scripts/test-verify-evidence.cjs` — zero-dependency fixture runner for the + evidence gate (26 cases), added to CI ## [4.0.0] - 2026-05-07 diff --git a/README.md b/README.md index da59a31..3feb453 100644 --- a/README.md +++ b/README.md @@ -161,15 +161,22 @@ In practice, devs skip steps for trivial work. The chains show the full discipli ## Development CI (`.github/workflows/validate.yml`) lints every skill against the 8-section -anatomy above, checks `plugin.json`/`marketplace.json` version sync, and -enforces token budgets on the always-loaded skill/agent frontmatter (the -"no agent-bloat" claim, measured). Run locally before committing: +anatomy above, checks `plugin.json`/`marketplace.json` version sync, enforces +token budgets on the always-loaded skill/agent frontmatter (the "no agent-bloat" +claim, measured), and exercises the evidence gate. Run locally before committing: ``` node scripts/validate-plugin.cjs node scripts/token-report.cjs --check +node scripts/test-verify-evidence.cjs ``` +`scripts/verify-evidence.cjs` is the evidence gate itself — dogfooding the +"every claim has evidence" philosophy against the agent's own output. It +resolves `file:line` citations in an artifact (`--citations `) and scans +`git diff HEAD` for fake-green tampering — deleted test files, added test skips, +new TODO/FIXME (`--tripwires`) — exiting non-zero when a claim can't be verified. + ## Requirements - Claude Code 1.0+ diff --git a/scripts/test-verify-evidence.cjs b/scripts/test-verify-evidence.cjs new file mode 100644 index 0000000..cb8403e --- /dev/null +++ b/scripts/test-verify-evidence.cjs @@ -0,0 +1,344 @@ +#!/usr/bin/env node +/** + * Test runner for verify-evidence.cjs (the evidence gate) and its fail-open + * hook wrapper verify-evidence-hook.cjs. Zero dependencies, no framework: + * builds hermetic fixtures in a temp dir, spawns the scripts, asserts exit + * codes. Exit 0 when every case passes, 1 otherwise. A missing script or + * spawn error counts as that case's FAIL — the run never aborts. + * + * Mirrors scripts/test-hooks.cjs conventions. + * + * Usage: node scripts/test-verify-evidence.cjs + */ +"use strict"; + +const { spawnSync } = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const GATE = path.join(__dirname, "verify-evidence.cjs"); +const HOOK = path.join(__dirname, "verify-evidence-hook.cjs"); + +let pass = 0; +let fail = 0; +const tmpRoots = []; + +function mkTmp() { + const d = fs.mkdtempSync(path.join(os.tmpdir(), "vee-")); + tmpRoots.push(d); + return d; +} + +function runGate(args, { cwd, diffFile, env } = {}) { + const e = { ...process.env, ...env }; + if (diffFile) e.VERIFY_EVIDENCE_DIFF_FILE = diffFile; + return spawnSync(process.execPath, [GATE, ...args], { + cwd: cwd || process.cwd(), + env: e, + encoding: "utf8", + timeout: 10000, + }); +} + +function runHook(stdin, { diffFile } = {}) { + const e = { ...process.env }; + if (diffFile) e.VERIFY_EVIDENCE_DIFF_FILE = diffFile; + return spawnSync(process.execPath, [HOOK], { + input: stdin, + env: e, + encoding: "utf8", + timeout: 10000, + }); +} + +function check(name, result, expected) { + let status = null; + let err = ""; + if (result.error) err = result.error.message; + else status = result.status; + const ok = !err && status === expected; + if (ok) { + pass++; + console.log(` ✓ ${name} (exit ${status})`); + } else { + fail++; + console.log(` ✗ ${name} — expected exit ${expected}, got ${err || `exit ${status}`}`); + } +} + +// ---- helpers to build fixtures ---- +function citationDir(artifactBody, realFileLines) { + const dir = mkTmp(); + if (realFileLines != null) { + fs.writeFileSync(path.join(dir, "real.js"), realFileLines.join("\n") + "\n"); + } + fs.writeFileSync(path.join(dir, "artifact.md"), artifactBody); + return dir; +} + +function diffFileWith(body) { + const dir = mkTmp(); + const f = path.join(dir, "captured.diff"); + fs.writeFileSync(f, body); + return f; +} + +// A minimal unified-diff builder. +function diffAdd(filePath, addedLines) { + return ( + `diff --git a/${filePath} b/${filePath}\n` + + `index 111..222 100644\n` + + `--- a/${filePath}\n` + + `+++ b/${filePath}\n` + + `@@ -1,1 +1,${addedLines.length + 1} @@\n` + + ` context\n` + + addedLines.map((l) => `+${l}`).join("\n") + + "\n" + ); +} + +function diffDelete(filePath) { + return ( + `diff --git a/${filePath} b/${filePath}\n` + + `deleted file mode 100644\n` + + `index 111..000\n` + + `--- a/${filePath}\n` + + `+++ /dev/null\n` + + `@@ -1,2 +0,0 @@\n` + + `-test('x', () => {});\n` + + `-test('y', () => {});\n` + ); +} + +function diffRename(fromPath, toPath) { + return ( + `diff --git a/${fromPath} b/${toPath}\n` + + `similarity index 100%\n` + + `rename from ${fromPath}\n` + + `rename to ${toPath}\n` + ); +} + +const diffConcat = (...parts) => parts.join(""); + +// ============ CITATION CASES ============ +console.log("\nverify-evidence.cjs --citations"); +{ + const real = ["line one", "line two", "line three"]; + + check( + "valid single-line citation", + runGate(["--citations", "artifact.md"], { + cwd: citationDir("Root caused in `real.js:2` per the trace.", real), + }), + 0 + ); + + check( + "valid range citation", + runGate(["--citations", "artifact.md"], { + cwd: citationDir("See real.js:1-3 for the fix.", real), + }), + 0 + ); + + check( + "out-of-range line", + runGate(["--citations", "artifact.md"], { + cwd: citationDir("Broken at real.js:999.", real), + }), + 1 + ); + + check( + "missing file", + runGate(["--citations", "artifact.md"], { + cwd: citationDir("See gone.js:1 for detail.", real), + }), + 1 + ); + + check( + "absolute path citation is a violation", + runGate(["--citations", "artifact.md"], { + cwd: citationDir("Config at /etc/hosts:1 shows it.", real), + }), + 1 + ); + + check( + "prose colons and URLs are not citations", + runGate(["--citations", "artifact.md"], { + cwd: citationDir( + "The ratio was 3:2 and we hit http://example.com:8080 at 12:30.", + real + ), + }), + 0 + ); + + check( + "decimal-colon prose (3.5:2) is not a citation", + runGate(["--citations", "artifact.md"], { + cwd: citationDir("Throughput rose 3.5:2 over baseline, and 18.04:30 held.", real), + }), + 0 + ); + + check( + "last-line+1 on a newline-terminated file is out of range", + runGate(["--citations", "artifact.md"], { + cwd: citationDir("See real.js:2 for the change.", ["only one line"]), + }), + 1 + ); + + check( + "no citations at all", + runGate(["--citations", "artifact.md"], { + cwd: citationDir("Just prose, nothing to resolve.", real), + }), + 0 + ); +} + +// ============ TRIPWIRE CASES ============ +console.log("\nverify-evidence.cjs --tripwires"); +{ + check( + "added .skip in a test file", + runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.test.js", ["it.skip('later', () => {});"])) }), + 1 + ); + check( + "added xit in a spec file", + runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.spec.js", ["xit('later', () => {});"])) }), + 1 + ); + check( + "added @pytest.mark.skip", + runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("tests/test_app.py", ["@pytest.mark.skip(reason='x')"])) }), + 1 + ); + check( + "added @unittest.skip", + runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("tests/test_app.py", ["@unittest.skip('x')"])) }), + 1 + ); + check( + "added TODO in changed code", + runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.js", ["// TODO: implement for real"])) }), + 1 + ); + check( + "added FIXME in changed code", + runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.js", ["# FIXME later"])) }), + 1 + ); + check( + "deleted test file", + runGate(["--tripwires"], { diffFile: diffFileWith(diffDelete("tests/test_app.py")) }), + 1 + ); + check( + "test renamed to another test path is fine", + runGate(["--tripwires"], { diffFile: diffFileWith(diffRename("tests/test_old.py", "tests/test_new.py")) }), + 0 + ); + check( + "test renamed OUT of the suite is a violation", + runGate(["--tripwires"], { diffFile: diffFileWith(diffRename("tests/test_app.py", "src/app.py")) }), + 1 + ); + check( + "added bare @skip decorator", + runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("tests/test_app.py", ["@skip"])) }), + 1 + ); + check( + "multi-file diff: clean file then skip in a later file, attributed correctly", + runGate(["--tripwires"], { + diffFile: diffFileWith( + diffConcat( + diffAdd("src/util.js", ["const y = 2;"]), + diffAdd("src/util.test.js", ["it.skip('later', () => {});"]) + ) + ), + }), + 1 + ); + check( + "multi-file diff: deleted non-test then clean adds does not false-flag", + runGate(["--tripwires"], { + diffFile: diffFileWith(diffConcat(diffDelete("src/legacy.js"), diffAdd("src/new.js", ["const z = 3;"]))), + }), + 0 + ); + check( + "clean diff of normal code", + runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.js", ["const x = 1;", "return x;"])) }), + 0 + ); + check( + "empty diff", + runGate(["--tripwires"], { diffFile: diffFileWith("") }), + 0 + ); + check( + "deleting a NON-test file is fine", + runGate(["--tripwires"], { diffFile: diffFileWith(diffDelete("src/legacy.js")) }), + 0 + ); + check( + "no git and no diff file -> skipped, not a failure", + runGate(["--tripwires"], { cwd: mkTmp() }), + 0 + ); +} + +// ============ COMBINED / NO-FLAG / HELP ============ +console.log("\nverify-evidence.cjs combined & meta"); +{ + const dir = citationDir("Broken at real.js:999.", ["a", "b"]); + check( + "combined: bad citation + skip tripwire -> exit 1", + runGate(["--citations", "artifact.md", "--tripwires"], { + cwd: dir, + diffFile: diffFileWith(diffAdd("a.test.js", ["it.skip('x', () => {})"])), + }), + 1 + ); + check( + "no flags runs tripwires (clean diff) -> exit 0", + runGate([], { cwd: mkTmp(), diffFile: diffFileWith("") }), + 0 + ); + check("--help exits 0", runGate(["--help"]), 0); +} + +// ============ HOOK (fail-open) ============ +console.log("\nverify-evidence-hook.cjs (always exit 0)"); +{ + const stop = JSON.stringify({ hook_event_name: "Stop" }); + check("valid stop event, clean diff", runHook(stop, { diffFile: diffFileWith("") }), 0); + check( + "violation present -> hook still exits 0 (advisory)", + runHook(stop, { diffFile: diffFileWith(diffAdd("a.test.js", ["it.skip('x', () => {})"])) }), + 0 + ); + check("malformed stdin -> exit 0", runHook("{not json", {}), 0); + check("empty stdin -> exit 0", runHook("", {}), 0); +} + +// ---- cleanup ---- +for (const d of tmpRoots) { + try { + fs.rmSync(d, { recursive: true, force: true }); + } catch { + /* best effort */ + } +} + +console.log(`\n${fail === 0 ? "OK" : "FAIL"} — ${pass} passed, ${fail} failed`); +process.exit(fail === 0 ? 0 : 1); diff --git a/scripts/verify-evidence-hook.cjs b/scripts/verify-evidence-hook.cjs new file mode 100644 index 0000000..b385802 --- /dev/null +++ b/scripts/verify-evidence-hook.cjs @@ -0,0 +1,46 @@ +#!/usr/bin/env node +/** + * Fail-open hook wrapper around the evidence gate (scripts/verify-evidence.cjs). + * Intended for a Stop / PostToolUse hook: after the agent claims work is done, + * run the tripwire scan and surface any fake-green findings as ADVISORY output + * — it never blocks and never exits non-zero on its own errors. + * + * Fails open (exit 0 always), per the repo's hook convention + * (see scripts/auto-format.cjs, scripts/detect-secrets.cjs). The loud, + * blocking enforcement lives in verify-evidence.cjs run as a CI gate; this + * wrapper is the low-friction in-session nudge. + */ +"use strict"; + +const path = require("path"); +const { spawnSync } = require("child_process"); + +async function main() { + try { + // Drain stdin (hook payload); content is not needed for the diff scan. + let data = ""; + for await (const chunk of process.stdin) data += chunk; + try { + JSON.parse(data); + } catch { + // Malformed / empty payload is fine — this hook fires opportunistically. + } + + const gate = path.join(__dirname, "verify-evidence.cjs"); + const r = spawnSync(process.execPath, [gate, "--tripwires"], { + encoding: "utf8", + timeout: 10000, + }); + + // Exit 1 from the gate means tripwires were found — surface them, advisory. + if (r.status === 1 && (r.stderr || r.stdout)) { + console.error("[evidence] fake-green tripwires detected (advisory):"); + console.error(r.stderr || r.stdout); + } + } catch { + // Fail open — a hook bug must never stall or block the session. + } + process.exit(0); +} + +main(); diff --git a/scripts/verify-evidence.cjs b/scripts/verify-evidence.cjs new file mode 100644 index 0000000..439292f --- /dev/null +++ b/scripts/verify-evidence.cjs @@ -0,0 +1,334 @@ +#!/usr/bin/env node +/** + * Evidence gate: mechanically checks that an AI agent's *claimed* evidence is + * real, rather than trusting the transcript. Two modes: + * + * --citations Resolve `file:line` / `file:start-end` citations + * in an evidence artifact; fail if a path is + * missing, absolute/outside the repo, or the line + * is out of range. + * --tripwires Scan `git diff HEAD` for fake-green tampering: + * deleted test files, newly skipped tests, and new + * TODO/FIXME left in changed code. + * + * With no mode flag, runs --tripwires (citations need an artifact argument). + * + * This is a GATE, not a convenience hook: it fails loud. + * exit 0 = clean exit 1 = violations found exit 2 = usage / internal error + * (Exit 2 is kept distinct from 1 so a crash can never masquerade as a + * "violation", per scripts/test-verify-evidence.cjs.) + * + * Precision-first by design: it would rather miss an oddly-shaped citation or an + * unusually-named test than cry wolf — a gate that cries wolf gets disabled. It + * deliberately does NOT attempt deep test-tamper analysis (assertion-count + * deltas, mutation proofs); that is out of scope for this slice. + * + * Zero dependencies. Usage: node scripts/verify-evidence.cjs [--citations ] [--tripwires] + */ +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const { spawnSync } = require("child_process"); + +const USAGE = + "Usage: node scripts/verify-evidence.cjs [--citations ] [--tripwires]"; + +// ---------------------------------------------------------------- citations +// A citation is `path:line` or `path:start-end`, backtick-optional. To avoid +// false positives from prose ("ratio 3:2", "12:30") and URLs, the path token +// must contain a "." or "/", URLs are stripped first, and a match preceded by a +// word char or ":" is rejected. +const CITATION_RE = + /(? lineCount) { + violations.push( + `${c.raw} — line out of range (file has ${lineCount} lines)` + ); + } + } + return { violations, checked: citations.length }; +} + +// ---------------------------------------------------------------- tripwires +function isTestFile(p) { + const f = p.toLowerCase(); + return ( + /(^|\/)tests?\//.test(f) || + /(^|\/)__tests__\//.test(f) || + /\.(test|spec)\.[a-z0-9]+$/.test(f) || + /(^|\/)test_[^/]*\.py$/.test(f) || + /_test\.py$/.test(f) || + // "test"/"spec" as a separator-bounded token in the basename — catches + // e.g. app.test.js, foo_spec.rb, but not latest.js / inspector.js. + /(^|[\/._-])(test|spec)([\/._-]|$)/.test(f) + ); +} + +const SKIP_PATTERNS = [ + { name: "skipped test (.skip)", re: /\.skip\s*\(/ }, + { name: "skipped test (xit/xdescribe/xtest)", re: /\bx(it|describe|test)\s*\(/ }, + // Matches @skip, @pytest.mark.skip, @pytest.mark.skipif, @unittest.skip, + // @unittest.skipUnless, @unittest.skipIf. + { name: "skipped test (@skip decorator)", re: /@(?:pytest\.mark\.|unittest\.)?skip/ }, +]; +const MARKER_RE = /\b(TODO|FIXME)\b/; + +function getDiff() { + const override = process.env.VERIFY_EVIDENCE_DIFF_FILE; + if (override) { + try { + return { text: fs.readFileSync(override, "utf8") }; + } catch { + return { unavailable: `cannot read VERIFY_EVIDENCE_DIFF_FILE: ${override}` }; + } + } + const r = spawnSync("git", ["diff", "HEAD"], { encoding: "utf8", timeout: 10000 }); + if (r.error || r.status !== 0) { + return { unavailable: "git diff HEAD unavailable (not a git repo?)" }; + } + return { text: r.stdout || "" }; +} + +function checkTripwires() { + const diff = getDiff(); + if (diff.unavailable) return { skipped: diff.unavailable }; + + const violations = []; + const lines = diff.text.split("\n"); + let aPath = null; + let bPath = null; + let curFile = null; + let newLineNo = 0; + let renameFrom = null; + + const stripPrefix = (s) => + s === "/dev/null" ? "/dev/null" : s.replace(/^[ab]\//, ""); + + for (const line of lines) { + // A new file section resets all per-file state, so location and deletion + // attribution never leak across files in a multi-file diff. + if (line.startsWith("diff --git ")) { + aPath = null; + bPath = null; + curFile = null; + newLineNo = 0; + renameFrom = null; + continue; + } + if (line.startsWith("rename from ")) { + renameFrom = line.slice("rename from ".length).trim(); + continue; + } + if (line.startsWith("rename to ")) { + const renameTo = line.slice("rename to ".length).trim(); + // Renaming a test file to a non-test path removes it from the suite. + if (renameFrom && isTestFile(renameFrom) && !isTestFile(renameTo)) { + violations.push(`test file renamed out of the suite: ${renameFrom} → ${renameTo}`); + } + continue; + } + if (line.startsWith("--- ")) { + aPath = stripPrefix(line.slice(4).trim()); + continue; + } + if (line.startsWith("+++ ")) { + bPath = stripPrefix(line.slice(4).trim()); + curFile = bPath === "/dev/null" ? aPath : bPath; + // A file whose new side is /dev/null was deleted. + if (bPath === "/dev/null" && aPath && isTestFile(aPath)) { + violations.push(`deleted test file: ${aPath}`); + } + continue; + } + const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line); + if (hunk) { + newLineNo = parseInt(hunk[1], 10); + continue; + } + // Added content line (not the "+++ " file header, handled above). + if (line.startsWith("+")) { + const content = line.slice(1); + const loc = curFile ? `${curFile}:${newLineNo}` : "(unknown)"; + for (const s of SKIP_PATTERNS) { + if (s.re.test(content)) violations.push(`${s.name} added at ${loc}`); + } + if (MARKER_RE.test(content)) { + violations.push(`TODO/FIXME added at ${loc}: ${content.trim()}`); + } + newLineNo++; + continue; + } + if (line.startsWith(" ")) { + newLineNo++; + continue; + } + // "-" lines and metadata: no new-side advance + } + return { violations }; +} + +// ---------------------------------------------------------------- main +function main() { + const argv = process.argv.slice(2); + if (argv.includes("--help") || argv.includes("-h")) { + console.log(USAGE); + return 0; + } + + let artifact = null; + let doCitations = false; + let doTripwires = false; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === "--citations") { + doCitations = true; + artifact = argv[i + 1] && !argv[i + 1].startsWith("-") ? argv[++i] : null; + } else if (argv[i] === "--tripwires") { + doTripwires = true; + } else { + console.error(`Unknown argument: ${argv[i]}\n${USAGE}`); + return 2; + } + } + // Default: no mode flag → tripwires (citations require an artifact arg). + if (!doCitations && !doTripwires) doTripwires = true; + + const findings = []; + let anyViolation = false; + + if (doCitations) { + if (!artifact) { + console.error(`--citations requires an artifact path\n${USAGE}`); + return 2; + } + const res = checkCitations(artifact); + if (res.usageError) { + console.error(res.usageError); + return 2; + } + if (res.violations.length) { + anyViolation = true; + findings.push( + `citations: ${res.violations.length} unresolved of ${res.checked} checked` + ); + for (const v of res.violations) findings.push(` ✗ ${v}`); + } else { + findings.push(`citations: ${res.checked} checked, all resolve`); + } + } + + if (doTripwires) { + const res = checkTripwires(); + if (res.skipped) { + findings.push(`tripwires: skipped — ${res.skipped}`); + } else if (res.violations.length) { + anyViolation = true; + findings.push(`tripwires: ${res.violations.length} found`); + for (const v of res.violations) findings.push(` ✗ ${v}`); + } else { + findings.push(`tripwires: none`); + } + } + + const out = findings.join("\n"); + if (anyViolation) { + console.error(out); + console.error("\nFAIL — evidence gate found unverifiable claims."); + return 1; + } + console.log(out); + console.log("\nOK — evidence gate passed."); + return 0; +} + +try { + process.exit(main()); +} catch (err) { + // Internal error is exit 2, never 1 — a crash must not read as a violation. + console.error(`verify-evidence: internal error: ${err && err.message}`); + process.exit(2); +}