diff --git a/AGENTS.md b/AGENTS.md index b53eb8a..ae5a201 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,8 +13,9 @@ docs website (`website/`, Astro Starlight). The only repo-level check is — validates every SKILL.md against the 8-section anatomy and plugin/marketplace version sync; `node scripts/token-report.cjs --check` — enforces token budgets on skill/agent frontmatter; `node scripts/test-verify-evidence.cjs` — exercises - the evidence gate (`scripts/verify-evidence.cjs`: citation resolution + - fake-green tripwires). All three run in CI, `.github/workflows/validate.yml`. + the evidence gate (`scripts/verify-evidence.cjs`: citation resolution, + fake-green tripwires, and `--rerun` claim-vs-actual test diffing). All three + run in CI, `.github/workflows/validate.yml`. - Test by installing locally: `/plugin marketplace add ` → `/plugin install claudekit` → `/claudekit:init` - Website (`cd website/`): `npm install`, `npm run dev`, `npm run build`, diff --git a/CHANGELOG.md b/CHANGELOG.md index 5820faf..6440d67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,8 +32,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 does not attempt deep test-tamper analysis - `scripts/verify-evidence-hook.cjs` — fail-open Stop/PostToolUse wrapper that surfaces tripwire findings as advisory, never blocking +- `scripts/verify-evidence.cjs --rerun ` — re-runs the test suite and + diffs the actual result against the result claimed in an evidence artifact + (claimed pass but red, or divergent pass/fail counts, fails); verdict is + ground-truthed from the run's exit code. Command auto-detected (`npm test` / + `pytest`) or via `--cmd`; `--detect-only` prints the detected command. Not + wired into the fail-open hook (too heavy per turn) - `scripts/test-verify-evidence.cjs` — zero-dependency fixture runner for the - evidence gate (26 cases), added to CI + evidence gate (49 cases), run in CI ## [4.0.0] - 2026-05-07 diff --git a/README.md b/README.md index 3feb453..09bc040 100644 --- a/README.md +++ b/README.md @@ -172,10 +172,20 @@ node scripts/test-verify-evidence.cjs ``` `scripts/verify-evidence.cjs` is the evidence gate itself — dogfooding the -"every claim has evidence" philosophy against the agent's own output. It -resolves `file:line` citations in an artifact (`--citations `) and scans -`git diff HEAD` for fake-green tampering — deleted test files, added test skips, -new TODO/FIXME (`--tripwires`) — exiting non-zero when a claim can't be verified. +"every claim has evidence" philosophy against the agent's own output: + +- `--citations ` — resolves `file:line` citations in an artifact against + real paths and line ranges. +- `--tripwires` — scans `git diff HEAD` for fake-green tampering: deleted test + files, added test skips, new TODO/FIXME. +- `--rerun [--cmd ""]` — re-runs the test suite and diffs the + actual result against the result *claimed* in the artifact; a claimed pass + that's actually red, or divergent pass/fail counts, fails the gate. The + verdict is ground-truthed from the run's exit code, so a pasted output block + can't fake green. The command is auto-detected (`npm test` / `pytest`) or + given with `--cmd`; `--detect-only` prints what would run. + +It exits non-zero when a claim can't be verified. ## Requirements diff --git a/scripts/test-verify-evidence.cjs b/scripts/test-verify-evidence.cjs index cb8403e..7d299eb 100644 --- a/scripts/test-verify-evidence.cjs +++ b/scripts/test-verify-evidence.cjs @@ -122,6 +122,39 @@ function diffRename(fromPath, toPath) { const diffConcat = (...parts) => parts.join(""); +// ---- rerun helpers ---- +// A fake test runner: a node script that prints a summary line and exits with a +// chosen code, so claim-vs-actual comparison is fully hermetic. +function fakeRunner(summaryLine, code) { + const dir = mkTmp(); + const f = path.join(dir, "runner.cjs"); + fs.writeFileSync( + f, + `process.stdout.write(${JSON.stringify(summaryLine + "\n")});\n` + + `process.exit(${code});\n` + ); + return `${JSON.stringify(process.execPath)} ${JSON.stringify(f)}`; +} +function claimArtifact(body) { + const dir = mkTmp(); + const f = path.join(dir, "claim.md"); + fs.writeFileSync(f, body); + return f; +} +function npmProjectDir(testScript) { + const dir = mkTmp(); + fs.writeFileSync( + path.join(dir, "package.json"), + JSON.stringify({ name: "x", scripts: { test: testScript } }) + ); + return dir; +} +function pytestDir() { + const dir = mkTmp(); + fs.writeFileSync(path.join(dir, "pytest.ini"), "[pytest]\n"); + return dir; +} + // ============ CITATION CASES ============ console.log("\nverify-evidence.cjs --citations"); { @@ -331,6 +364,159 @@ console.log("\nverify-evidence-hook.cjs (always exit 0)"); check("empty stdin -> exit 0", runHook("", {}), 0); } +// ============ RERUN CASES ============ +console.log("\nverify-evidence.cjs --rerun"); +{ + // AC1 — claim matches a passing run with matching counts. + check( + "claim pass + counts match a passing run", + runGate(["--rerun", claimArtifact("Verified: 42 passed, 0 failed."), "--cmd", fakeRunner("42 passed, 0 failed", 0)]), + 0 + ); + // AC2 — claimed pass but the run fails (non-zero exit). + check( + "claimed pass but suite fails (exit 1)", + runGate(["--rerun", claimArtifact("All tests pass."), "--cmd", fakeRunner("1 failed, 41 passed", 1)]), + 1 + ); + // AC3 — claimed count differs from actual passed count. + check( + "claimed 42 passed but actual reports 12", + runGate(["--rerun", claimArtifact("Verified: 42 passed, 0 failed."), "--cmd", fakeRunner("12 passed, 0 failed", 0)]), + 1 + ); + // AC4 — no parseable claim in the artifact. + check( + "artifact has no test claim -> nothing to verify", + runGate(["--rerun", claimArtifact("Refactored the parser; see notes."), "--cmd", fakeRunner("5 passed, 0 failed", 0)]), + 0 + ); + // AC5 — no --cmd and no detectable command -> cannot verify (exit 2). + check( + "no --cmd and no detectable command -> exit 2", + runGate(["--rerun", claimArtifact("42 passed, 0 failed.")], { cwd: mkTmp() }), + 2 + ); + // A7 — claimed FAIL but the suite actually passes -> mismatch. + check( + "claimed failures but suite passes -> violation", + runGate(["--rerun", claimArtifact("Result: 3 failed, 39 passed."), "--cmd", fakeRunner("42 passed, 0 failed", 0)]), + 1 + ); + // G1 — "N failed, M passed" order: claimed FAIL detected even when counts + // would otherwise agree (the failed count is not dropped). + check( + "claimed '3 failed, 39 passed' vs actual '39 passed' -> violation via verdict", + runGate(["--rerun", claimArtifact("Summary: 3 failed, 39 passed."), "--cmd", fakeRunner("39 passed, 0 failed", 0)]), + 1 + ); + // G1 — jest 'Tests:' line, failed-before-passed order parses both counts. + check( + "jest 'Tests: 2 failed, 40 passed' claim vs matching run -> violation (claimed fail)", + runGate(["--rerun", claimArtifact("Tests: 2 failed, 40 passed"), "--cmd", fakeRunner("Tests: 2 failed, 40 passed", 1)]), + 0 + ); + // G3 — "42 passedness" is not a passed count -> no claim. + check( + "'passedness' is not a test claim", + runGate(["--rerun", claimArtifact("The 42 passedness metric improved."), "--cmd", fakeRunner("7 passed, 0 failed", 0)]), + 0 + ); + // A2 — timeout / non-zero labelled "timed out" is treated as a failed run. + check( + "claimed pass but runner exits non-zero with no counts -> violation", + runGate(["--rerun", claimArtifact("Suite green."), "--cmd", fakeRunner("boom", 2)]), + 1 + ); + // A3/R2 — framework summary parsed, prose does not false-match generic. + check( + "jest-style actual summary compared to jest-style claim", + runGate(["--rerun", claimArtifact("Tests: 0 failed, 42 passed"), "--cmd", fakeRunner("Tests: 0 failed, 42 passed", 0)]), + 0 + ); + check( + "prose 'passed the review' is not a test claim", + runGate(["--rerun", claimArtifact("The change passed the review with flying colors."), "--cmd", fakeRunner("7 passed, 0 failed", 0)]), + 0 + ); +} + +// ============ DETECT-ONLY CASES ============ +console.log("\nverify-evidence.cjs --detect-only"); +{ + check( + "npm project (scripts.test) -> detects, exit 0", + runGate(["--detect-only"], { cwd: npmProjectDir("jest") }), + 0 + ); + check( + "pytest.ini marker -> detects, exit 0", + runGate(["--detect-only"], { cwd: pytestDir() }), + 0 + ); + check( + "bare dir (no markers) -> none detected, exit 2", + runGate(["--detect-only"], { cwd: mkTmp() }), + 2 + ); +} + +// ============ RERUN CLI VALIDATION (R6) ============ +console.log("\nverify-evidence.cjs --rerun CLI validation"); +{ + check( + "--cmd without --rerun -> exit 2", + runGate(["--cmd", "echo hi"]), + 2 + ); + check( + "--rerun with no artifact value -> exit 2", + runGate(["--rerun"]), + 2 + ); + check( + "--rerun with unreadable artifact -> exit 2", + runGate(["--rerun", "/no/such/artifact.md", "--cmd", fakeRunner("1 passed, 0 failed", 0)]), + 2 + ); + // G5 — a command that cannot be run (ENOENT) is exit 2, not a violation. + check( + "--cmd naming a missing binary -> exit 2 (cannot verify)", + runGate(["--rerun", claimArtifact("42 passed, 0 failed."), "--cmd", "definitely-not-a-real-binary-xyz-123"]), + 2 + ); + // G8 — --cmd with no value is a usage error, not a silent auto-detect. + check( + "--cmd with no value -> exit 2", + runGate(["--rerun", claimArtifact("1 passed, 0 failed."), "--cmd"]), + 2 + ); + // G9 — --detect-only --cmd is rejected (--cmd only pairs with --rerun). + check( + "--detect-only with --cmd -> exit 2", + runGate(["--detect-only", "--cmd", "npm test"], { cwd: npmProjectDir("jest") }), + 2 + ); + // aggregation: a citation violation AND a rerun run together still exit 1. + { + const dir = citationDir("Broken at real.js:999.", ["a", "b"]); + check( + "citation violation + clean rerun aggregate -> exit 1", + runGate([ + "--citations", path.join(dir, "artifact.md"), + "--rerun", claimArtifact("42 passed, 0 failed."), + "--cmd", fakeRunner("42 passed, 0 failed", 0), + ]), + 1 + ); + } + check( + "no-flag default still runs tripwires only (unchanged)", + runGate([], { cwd: mkTmp(), diffFile: diffFileWith("") }), + 0 + ); +} + // ---- cleanup ---- for (const d of tmpRoots) { try { diff --git a/scripts/verify-evidence.cjs b/scripts/verify-evidence.cjs index 439292f..e7be525 100644 --- a/scripts/verify-evidence.cjs +++ b/scripts/verify-evidence.cjs @@ -10,8 +10,14 @@ * --tripwires Scan `git diff HEAD` for fake-green tampering: * deleted test files, newly skipped tests, and new * TODO/FIXME left in changed code. + * --rerun Re-run the test suite and diff the ACTUAL result + * against the result CLAIMED in the artifact; fail + * if a claimed pass is actually red, or the counts + * diverge. Command auto-detected, or --cmd "". + * --detect-only Print the auto-detected test command (exit 2 if + * none); does not run anything. * - * With no mode flag, runs --tripwires (citations need an artifact argument). + * With no mode flag, runs --tripwires (citations/rerun need an artifact argument). * * This is a GATE, not a convenience hook: it fails loud. * exit 0 = clean exit 1 = violations found exit 2 = usage / internal error @@ -23,7 +29,7 @@ * deliberately does NOT attempt deep test-tamper analysis (assertion-count * deltas, mutation proofs); that is out of scope for this slice. * - * Zero dependencies. Usage: node scripts/verify-evidence.cjs [--citations ] [--tripwires] + * Zero dependencies. Usage: node scripts/verify-evidence.cjs [--citations ] [--tripwires] [--rerun [--cmd ""]] [--detect-only] */ "use strict"; @@ -32,7 +38,8 @@ const path = require("path"); const { spawnSync } = require("child_process"); const USAGE = - "Usage: node scripts/verify-evidence.cjs [--citations ] [--tripwires]"; + 'Usage: node scripts/verify-evidence.cjs [--citations ] ' + + '[--tripwires] [--rerun [--cmd ""]] [--detect-only]'; // ---------------------------------------------------------------- citations // A citation is `path:line` or `path:start-end`, backtick-optional. To avoid @@ -252,6 +259,176 @@ function checkTripwires() { return { violations }; } +// ---------------------------------------------------------------- rerun +// Last integer captured by a global regex (or null). Word-boundaried so +// "42 passedness" does not match "42 passed". +function lastCount(re, t) { + let m; + let val = null; + while ((m = re.exec(t)) !== null) val = parseInt(m[1], 10); + return val; +} + +// Parse a claimed/actual test summary: {passed, failed, verdict}. passed/failed +// are null when not stated; verdict is "pass" | "fail" | null. Counts are read +// order-independently ("3 failed, 39 passed" and "39 passed, 3 failed" both +// yield passed:39 failed:3) so a claimed failure is never lost. +function parseTestSummary(text) { + const t = String(text); + let passed = null; + let failed = null; + + // mocha uses "passing"/"failing"; jest/vitest/pytest/generic use + // "passed"/"failed". They never collide, so we scan for each independently + // (last occurrence wins) — order-independent, and the real final summary line + // wins over any earlier per-file or sample numbers. + passed = lastCount(/\b(\d+)\s+passing\b/gi, t); + failed = lastCount(/\b(\d+)\s+failing\b/gi, t); + if (passed == null) passed = lastCount(/\b(\d+)\s+passed\b/gi, t); + if (failed == null) failed = lastCount(/\b(\d+)\s+failed\b/gi, t); + + let verdict = null; + if (failed != null && failed > 0) verdict = "fail"; + else if (failed === 0 || passed != null) verdict = "pass"; + else if (/\b(all tests? pass|tests? pass|suite green|all green)\b/i.test(t)) { + verdict = "pass"; + } + return { passed, failed, verdict }; +} + +// Detect the repo's test command. Returns {argv, label} or null. Precision- +// first: only a package.json test script or an explicit pytest config marker. +function detectTestCommand(cwd) { + try { + const pkgPath = path.join(cwd, "package.json"); + if (fs.existsSync(pkgPath)) { + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")); + if (pkg && pkg.scripts && typeof pkg.scripts.test === "string") { + return { argv: ["npm", "test"], label: "npm test" }; + } + } + } catch { + /* unreadable package.json → fall through */ + } + const hasPytest = + fs.existsSync(path.join(cwd, "pytest.ini")) || + /\[tool:pytest\]|\[pytest\]/.test(safeRead(path.join(cwd, "tox.ini"))) || + /\[tool:pytest\]|\[pytest\]/.test(safeRead(path.join(cwd, "setup.cfg"))) || + /\[tool\.pytest/.test(safeRead(path.join(cwd, "pyproject.toml"))); + if (hasPytest) return { argv: ["pytest"], label: "pytest" }; + return null; +} + +function safeRead(p) { + try { + return fs.readFileSync(p, "utf8"); + } catch { + return ""; + } +} + +// Run the resolved command. Returns {code, output, label} or {enoent:true}. +// A timeout or signal termination is reported as a non-zero "timed out" result. +function runTests(resolved) { + const opts = { encoding: "utf8", timeout: 120000, maxBuffer: 10 * 1024 * 1024 }; + let r; + if (resolved.shellCmd) { + r = spawnSync(resolved.shellCmd, { ...opts, shell: true }); + } else { + r = spawnSync(resolved.argv[0], resolved.argv.slice(1), opts); + } + const output = `${r.stdout || ""}\n${r.stderr || ""}`; + // A timeout or a signal termination is a non-zero "terminated" run (A2). + if ((r.error && r.error.code === "ETIMEDOUT") || r.signal) { + const why = r.signal ? `terminated: ${r.signal}` : "timed out"; + return { code: 124, output: `${output}\n[${why}]`, timedOut: true }; + } + // Any other spawn error (ENOENT, EACCES, …) means the command could not run → + // can't-verify, not a violation (the caller maps this to exit 2). + if (r.error) return { cannotRun: r.error.code || r.error.message }; + // Shell "command not found" (127) / "not executable" (126) are also can't-run, + // not a test failure — a missing --cmd binary must not read as a red suite. + if (r.status === 127) return { cannotRun: "command not found (127)" }; + if (r.status === 126) return { cannotRun: "command not executable (126)" }; + return { code: r.status == null ? 1 : r.status, output }; +} + +function checkRerun(artifactArg, cmdOverride) { + let artifactText; + try { + artifactText = fs.readFileSync(artifactArg, "utf8"); + } catch { + return { usageError: `cannot read artifact: ${artifactArg}` }; + } + const claim = parseTestSummary(artifactText); + if (!claim.verdict) return { noClaim: true }; + + // Resolve the command: explicit override (shell) or auto-detect (argv). + let resolved; + let label; + if (cmdOverride) { + resolved = { shellCmd: cmdOverride }; + label = cmdOverride; + } else { + const det = detectTestCommand(process.cwd()); + if (!det) { + return { + usageError: + "no test command: none detected (need package.json test script or " + + "pytest config) — pass --cmd \"\"", + }; + } + resolved = { argv: det.argv }; + label = det.label; + } + + // Print the command BEFORE running it (D2): if the run hangs or is killed, + // the user still sees what was executed. + console.log(`rerun: running \`${label}\` …`); + const run = runTests(resolved); + if (run.cannotRun) { + return { usageError: `could not run test command \`${label}\` (${run.cannotRun})` }; + } + const actual = parseTestSummary(run.output); + const actualPass = run.code === 0; + + const violations = []; + const claimStr = `${claim.passed != null ? claim.passed + " passed" : "pass"}${ + claim.failed != null ? ", " + claim.failed + " failed" : "" + }`; + if (claim.verdict === "pass" && !actualPass) { + violations.push( + `artifact claims ${claimStr}, but the suite ${ + run.timedOut ? "timed out" : `failed (exit ${run.code})` + }` + ); + } else if (claim.verdict === "fail" && actualPass) { + violations.push( + `artifact claims ${claimStr}, but the suite passed (exit 0)` + ); + } + // Count divergence, when both sides are parseable. + if ( + claim.passed != null && + actual.passed != null && + claim.passed !== actual.passed + ) { + violations.push( + `claimed ${claim.passed} passed, actual run reported ${actual.passed} passed` + ); + } + if ( + claim.failed != null && + actual.failed != null && + claim.failed !== actual.failed + ) { + violations.push( + `claimed ${claim.failed} failed, actual run reported ${actual.failed} failed` + ); + } + return { violations, label, actual, actualPass }; +} + // ---------------------------------------------------------------- main function main() { const argv = process.argv.slice(2); @@ -261,21 +438,60 @@ function main() { } let artifact = null; + let rerunArtifact = null; + let cmdOverride = null; + let cmdSeen = false; let doCitations = false; let doTripwires = false; + let doRerun = false; + let doDetectOnly = false; + const takesValue = (v) => v && !v.startsWith("-"); for (let i = 0; i < argv.length; i++) { if (argv[i] === "--citations") { doCitations = true; - artifact = argv[i + 1] && !argv[i + 1].startsWith("-") ? argv[++i] : null; + artifact = takesValue(argv[i + 1]) ? argv[++i] : null; } else if (argv[i] === "--tripwires") { doTripwires = true; + } else if (argv[i] === "--rerun") { + doRerun = true; + rerunArtifact = takesValue(argv[i + 1]) ? argv[++i] : null; + } else if (argv[i] === "--cmd") { + cmdSeen = true; + cmdOverride = takesValue(argv[i + 1]) ? argv[++i] : null; + } else if (argv[i] === "--detect-only") { + doDetectOnly = true; } else { console.error(`Unknown argument: ${argv[i]}\n${USAGE}`); return 2; } } - // Default: no mode flag → tripwires (citations require an artifact arg). - if (!doCitations && !doTripwires) doTripwires = true; + + // Validate --cmd before any mode runs: it needs a value and only pairs with + // --rerun (checked here so --detect-only --cmd is also rejected). + if (cmdSeen && cmdOverride == null) { + console.error(`--cmd requires a command string\n${USAGE}`); + return 2; + } + if (cmdSeen && !doRerun) { + console.error(`--cmd is only valid with --rerun\n${USAGE}`); + return 2; + } + + // --detect-only: print the detected command (or exit 2 if none). No execution. + if (doDetectOnly) { + const det = detectTestCommand(process.cwd()); + if (!det) { + console.error( + "detect-only: no test command detected (need a package.json test " + + 'script or pytest config). Pass --cmd "" to override.' + ); + return 2; + } + console.log(`detected test command: ${det.label}`); + return 0; + } + // Default: no mode flag → tripwires (citations/rerun require an artifact arg). + if (!doCitations && !doTripwires && !doRerun) doTripwires = true; const findings = []; let anyViolation = false; @@ -314,6 +530,30 @@ function main() { } } + if (doRerun) { + if (!rerunArtifact) { + console.error(`--rerun requires an artifact path\n${USAGE}`); + return 2; + } + const res = checkRerun(rerunArtifact, cmdOverride); + if (res.usageError) { + console.error(res.usageError); + return 2; + } + if (res.noClaim) { + findings.push("rerun: no test claim found in artifact — nothing to verify"); + } else { + findings.push(`rerun: ran \`${res.label}\` → ${res.actualPass ? "passed" : "failed"}`); + if (res.violations.length) { + anyViolation = true; + findings.push(`rerun: ${res.violations.length} claim mismatch(es)`); + for (const v of res.violations) findings.push(` ✗ ${v}`); + } else { + findings.push(`rerun: claim matches the actual run`); + } + } + } + const out = findings.join("\n"); if (anyViolation) { console.error(out);