feat: evidence gate --rerun — diff actual test run against the artifact's claim (#5)

AGENTS.md

@@ -13,8 +13,9 @@ docs website (`website/`, Astro Starlight). The only repo-level check is
   — validates every SKILL.md against the 8-section anatomy and plugin/marketplace
   version sync; `node scripts/token-report.cjs --check` — enforces token budgets
   on skill/agent frontmatter; `node scripts/test-verify-evidence.cjs` — exercises
-  the evidence gate (`scripts/verify-evidence.cjs`: citation resolution +
+  the evidence gate (`scripts/verify-evidence.cjs`: citation resolution,
-  fake-green tripwires). All three run in CI, `.github/workflows/validate.yml`.
+  fake-green tripwires, and `--rerun` claim-vs-actual test diffing). All three
+  run in CI, `.github/workflows/validate.yml`.
 - Test by installing locally:
   `/plugin marketplace add <path-or-repo>` → `/plugin install claudekit` → `/claudekit:init`
 - Website (`cd website/`): `npm install`, `npm run dev`, `npm run build`,

CHANGELOG.md

@@ -32,8 +32,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   does not attempt deep test-tamper analysis
 - `scripts/verify-evidence-hook.cjs` — fail-open Stop/PostToolUse wrapper that
   surfaces tripwire findings as advisory, never blocking
+- `scripts/verify-evidence.cjs --rerun <artifact>` — re-runs the test suite and
+  diffs the actual result against the result claimed in an evidence artifact
+  (claimed pass but red, or divergent pass/fail counts, fails); verdict is
+  ground-truthed from the run's exit code. Command auto-detected (`npm test` /
+  `pytest`) or via `--cmd`; `--detect-only` prints the detected command. Not
+  wired into the fail-open hook (too heavy per turn)
 - `scripts/test-verify-evidence.cjs` — zero-dependency fixture runner for the
-  evidence gate (26 cases), added to CI
+  evidence gate (49 cases), run in CI
 
 ## [4.0.0] - 2026-05-07
 

README.md

@@ -172,10 +172,20 @@ node scripts/test-verify-evidence.cjs
 ```
 
 `scripts/verify-evidence.cjs` is the evidence gate itself — dogfooding the
-"every claim has evidence" philosophy against the agent's own output. It
+"every claim has evidence" philosophy against the agent's own output:
-resolves `file:line` citations in an artifact (`--citations <file>`) and scans
+
-`git diff HEAD` for fake-green tampering — deleted test files, added test skips,
+- `--citations <file>` — resolves `file:line` citations in an artifact against
-new TODO/FIXME (`--tripwires`) — exiting non-zero when a claim can't be verified.
+  real paths and line ranges.
+- `--tripwires` — scans `git diff HEAD` for fake-green tampering: deleted test
+  files, added test skips, new TODO/FIXME.
+- `--rerun <file> [--cmd "<command>"]` — re-runs the test suite and diffs the
+  actual result against the result *claimed* in the artifact; a claimed pass
+  that's actually red, or divergent pass/fail counts, fails the gate. The
+  verdict is ground-truthed from the run's exit code, so a pasted output block
+  can't fake green. The command is auto-detected (`npm test` / `pytest`) or
+  given with `--cmd`; `--detect-only` prints what would run.
+
+It exits non-zero when a claim can't be verified.
 
 ## Requirements
 

scripts/test-verify-evidence.cjs

@@ -122,6 +122,39 @@ function diffRename(fromPath, toPath) {
 
 const diffConcat = (...parts) => parts.join("");
 
+// ---- rerun helpers ----
+// A fake test runner: a node script that prints a summary line and exits with a
+// chosen code, so claim-vs-actual comparison is fully hermetic.
+function fakeRunner(summaryLine, code) {
+  const dir = mkTmp();
+  const f = path.join(dir, "runner.cjs");
+  fs.writeFileSync(
+    f,
+    `process.stdout.write(${JSON.stringify(summaryLine + "\n")});\n` +
+      `process.exit(${code});\n`
+  );
+  return `${JSON.stringify(process.execPath)} ${JSON.stringify(f)}`;
+}
+function claimArtifact(body) {
+  const dir = mkTmp();
+  const f = path.join(dir, "claim.md");
+  fs.writeFileSync(f, body);
+  return f;
+}
+function npmProjectDir(testScript) {
+  const dir = mkTmp();
+  fs.writeFileSync(
+    path.join(dir, "package.json"),
+    JSON.stringify({ name: "x", scripts: { test: testScript } })
+  );
+  return dir;
+}
+function pytestDir() {
+  const dir = mkTmp();
+  fs.writeFileSync(path.join(dir, "pytest.ini"), "[pytest]\n");
+  return dir;
+}
+
 // ============ CITATION CASES ============
 console.log("\nverify-evidence.cjs --citations");
 {
@@ -331,6 +364,159 @@ console.log("\nverify-evidence-hook.cjs (always exit 0)");
   check("empty stdin -> exit 0", runHook("", {}), 0);
 }
 
+// ============ RERUN CASES ============
+console.log("\nverify-evidence.cjs --rerun");
+{
+  // AC1 — claim matches a passing run with matching counts.
+  check(
+    "claim pass + counts match a passing run",
+    runGate(["--rerun", claimArtifact("Verified: 42 passed, 0 failed."), "--cmd", fakeRunner("42 passed, 0 failed", 0)]),
+    0
+  );
+  // AC2 — claimed pass but the run fails (non-zero exit).
+  check(
+    "claimed pass but suite fails (exit 1)",
+    runGate(["--rerun", claimArtifact("All tests pass."), "--cmd", fakeRunner("1 failed, 41 passed", 1)]),
+    1
+  );
+  // AC3 — claimed count differs from actual passed count.
+  check(
+    "claimed 42 passed but actual reports 12",
+    runGate(["--rerun", claimArtifact("Verified: 42 passed, 0 failed."), "--cmd", fakeRunner("12 passed, 0 failed", 0)]),
+    1
+  );
+  // AC4 — no parseable claim in the artifact.
+  check(
+    "artifact has no test claim -> nothing to verify",
+    runGate(["--rerun", claimArtifact("Refactored the parser; see notes."), "--cmd", fakeRunner("5 passed, 0 failed", 0)]),
+    0
+  );
+  // AC5 — no --cmd and no detectable command -> cannot verify (exit 2).
+  check(
+    "no --cmd and no detectable command -> exit 2",
+    runGate(["--rerun", claimArtifact("42 passed, 0 failed.")], { cwd: mkTmp() }),
+    2
+  );
+  // A7 — claimed FAIL but the suite actually passes -> mismatch.
+  check(
+    "claimed failures but suite passes -> violation",
+    runGate(["--rerun", claimArtifact("Result: 3 failed, 39 passed."), "--cmd", fakeRunner("42 passed, 0 failed", 0)]),
+    1
+  );
+  // G1 — "N failed, M passed" order: claimed FAIL detected even when counts
+  // would otherwise agree (the failed count is not dropped).
+  check(
+    "claimed '3 failed, 39 passed' vs actual '39 passed' -> violation via verdict",
+    runGate(["--rerun", claimArtifact("Summary: 3 failed, 39 passed."), "--cmd", fakeRunner("39 passed, 0 failed", 0)]),
+    1
+  );
+  // G1 — jest 'Tests:' line, failed-before-passed order parses both counts.
+  check(
+    "jest 'Tests: 2 failed, 40 passed' claim vs matching run -> violation (claimed fail)",
+    runGate(["--rerun", claimArtifact("Tests: 2 failed, 40 passed"), "--cmd", fakeRunner("Tests: 2 failed, 40 passed", 1)]),
+    0
+  );
+  // G3 — "42 passedness" is not a passed count -> no claim.
+  check(
+    "'passedness' is not a test claim",
+    runGate(["--rerun", claimArtifact("The 42 passedness metric improved."), "--cmd", fakeRunner("7 passed, 0 failed", 0)]),
+    0
+  );
+  // A2 — timeout / non-zero labelled "timed out" is treated as a failed run.
+  check(
+    "claimed pass but runner exits non-zero with no counts -> violation",
+    runGate(["--rerun", claimArtifact("Suite green."), "--cmd", fakeRunner("boom", 2)]),
+    1
+  );
+  // A3/R2 — framework summary parsed, prose does not false-match generic.
+  check(
+    "jest-style actual summary compared to jest-style claim",
+    runGate(["--rerun", claimArtifact("Tests: 0 failed, 42 passed"), "--cmd", fakeRunner("Tests: 0 failed, 42 passed", 0)]),
+    0
+  );
+  check(
+    "prose 'passed the review' is not a test claim",
+    runGate(["--rerun", claimArtifact("The change passed the review with flying colors."), "--cmd", fakeRunner("7 passed, 0 failed", 0)]),
+    0
+  );
+}
+
+// ============ DETECT-ONLY CASES ============
+console.log("\nverify-evidence.cjs --detect-only");
+{
+  check(
+    "npm project (scripts.test) -> detects, exit 0",
+    runGate(["--detect-only"], { cwd: npmProjectDir("jest") }),
+    0
+  );
+  check(
+    "pytest.ini marker -> detects, exit 0",
+    runGate(["--detect-only"], { cwd: pytestDir() }),
+    0
+  );
+  check(
+    "bare dir (no markers) -> none detected, exit 2",
+    runGate(["--detect-only"], { cwd: mkTmp() }),
+    2
+  );
+}
+
+// ============ RERUN CLI VALIDATION (R6) ============
+console.log("\nverify-evidence.cjs --rerun CLI validation");
+{
+  check(
+    "--cmd without --rerun -> exit 2",
+    runGate(["--cmd", "echo hi"]),
+    2
+  );
+  check(
+    "--rerun with no artifact value -> exit 2",
+    runGate(["--rerun"]),
+    2
+  );
+  check(
+    "--rerun with unreadable artifact -> exit 2",
+    runGate(["--rerun", "/no/such/artifact.md", "--cmd", fakeRunner("1 passed, 0 failed", 0)]),
+    2
+  );
+  // G5 — a command that cannot be run (ENOENT) is exit 2, not a violation.
+  check(
+    "--cmd naming a missing binary -> exit 2 (cannot verify)",
+    runGate(["--rerun", claimArtifact("42 passed, 0 failed."), "--cmd", "definitely-not-a-real-binary-xyz-123"]),
+    2
+  );
+  // G8 — --cmd with no value is a usage error, not a silent auto-detect.
+  check(
+    "--cmd with no value -> exit 2",
+    runGate(["--rerun", claimArtifact("1 passed, 0 failed."), "--cmd"]),
+    2
+  );
+  // G9 — --detect-only --cmd is rejected (--cmd only pairs with --rerun).
+  check(
+    "--detect-only with --cmd -> exit 2",
+    runGate(["--detect-only", "--cmd", "npm test"], { cwd: npmProjectDir("jest") }),
+    2
+  );
+  // aggregation: a citation violation AND a rerun run together still exit 1.
+  {
+    const dir = citationDir("Broken at real.js:999.", ["a", "b"]);
+    check(
+      "citation violation + clean rerun aggregate -> exit 1",
+      runGate([
+        "--citations", path.join(dir, "artifact.md"),
+        "--rerun", claimArtifact("42 passed, 0 failed."),
+        "--cmd", fakeRunner("42 passed, 0 failed", 0),
+      ]),
+      1
+    );
+  }
+  check(
+    "no-flag default still runs tripwires only (unchanged)",
+    runGate([], { cwd: mkTmp(), diffFile: diffFileWith("") }),
+    0
+  );
+}
+
 // ---- cleanup ----
 for (const d of tmpRoots) {
   try {

scripts/verify-evidence.cjs

@@ -10,8 +10,14 @@
  *   --tripwires                Scan `git diff HEAD` for fake-green tampering:
  *                              deleted test files, newly skipped tests, and new
  *                              TODO/FIXME left in changed code.
+ *   --rerun <artifact.md>      Re-run the test suite and diff the ACTUAL result
+ *                              against the result CLAIMED in the artifact; fail
+ *                              if a claimed pass is actually red, or the counts
+ *                              diverge. Command auto-detected, or --cmd "<c>".
+ *   --detect-only              Print the auto-detected test command (exit 2 if
+ *                              none); does not run anything.
  *
- * With no mode flag, runs --tripwires (citations need an artifact argument).
+ * With no mode flag, runs --tripwires (citations/rerun need an artifact argument).
  *
  * This is a GATE, not a convenience hook: it fails loud.
  *   exit 0 = clean   exit 1 = violations found   exit 2 = usage / internal error
@@ -23,7 +29,7 @@
  * deliberately does NOT attempt deep test-tamper analysis (assertion-count
  * deltas, mutation proofs); that is out of scope for this slice.
  *
- * Zero dependencies. Usage: node scripts/verify-evidence.cjs [--citations <f>] [--tripwires]
+ * Zero dependencies. Usage: node scripts/verify-evidence.cjs [--citations <f>] [--tripwires] [--rerun <f> [--cmd "<c>"]] [--detect-only]
  */
 "use strict";
 
@@ -32,7 +38,8 @@ const path = require("path");
 const { spawnSync } = require("child_process");
 
 const USAGE =
-  "Usage: node scripts/verify-evidence.cjs [--citations <artifact.md>] [--tripwires]";
+  'Usage: node scripts/verify-evidence.cjs [--citations <artifact.md>] ' +
+  '[--tripwires] [--rerun <artifact.md> [--cmd "<command>"]] [--detect-only]';
 
 // ---------------------------------------------------------------- citations
 // A citation is `path:line` or `path:start-end`, backtick-optional. To avoid
@@ -252,6 +259,176 @@ function checkTripwires() {
   return { violations };
 }
 
+// ---------------------------------------------------------------- rerun
+// Last integer captured by a global regex (or null). Word-boundaried so
+// "42 passedness" does not match "42 passed".
+function lastCount(re, t) {
+  let m;
+  let val = null;
+  while ((m = re.exec(t)) !== null) val = parseInt(m[1], 10);
+  return val;
+}
+
+// Parse a claimed/actual test summary: {passed, failed, verdict}. passed/failed
+// are null when not stated; verdict is "pass" | "fail" | null. Counts are read
+// order-independently ("3 failed, 39 passed" and "39 passed, 3 failed" both
+// yield passed:39 failed:3) so a claimed failure is never lost.
+function parseTestSummary(text) {
+  const t = String(text);
+  let passed = null;
+  let failed = null;
+
+  // mocha uses "passing"/"failing"; jest/vitest/pytest/generic use
+  // "passed"/"failed". They never collide, so we scan for each independently
+  // (last occurrence wins) — order-independent, and the real final summary line
+  // wins over any earlier per-file or sample numbers.
+  passed = lastCount(/\b(\d+)\s+passing\b/gi, t);
+  failed = lastCount(/\b(\d+)\s+failing\b/gi, t);
+  if (passed == null) passed = lastCount(/\b(\d+)\s+passed\b/gi, t);
+  if (failed == null) failed = lastCount(/\b(\d+)\s+failed\b/gi, t);
+
+  let verdict = null;
+  if (failed != null && failed > 0) verdict = "fail";
+  else if (failed === 0 || passed != null) verdict = "pass";
+  else if (/\b(all tests? pass|tests? pass|suite green|all green)\b/i.test(t)) {
+    verdict = "pass";
+  }
+  return { passed, failed, verdict };
+}
+
+// Detect the repo's test command. Returns {argv, label} or null. Precision-
+// first: only a package.json test script or an explicit pytest config marker.
+function detectTestCommand(cwd) {
+  try {
+    const pkgPath = path.join(cwd, "package.json");
+    if (fs.existsSync(pkgPath)) {
+      const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8"));
+      if (pkg && pkg.scripts && typeof pkg.scripts.test === "string") {
+        return { argv: ["npm", "test"], label: "npm test" };
+      }
+    }
+  } catch {
+    /* unreadable package.json → fall through */
+  }
+  const hasPytest =
+    fs.existsSync(path.join(cwd, "pytest.ini")) ||
+    /\[tool:pytest\]|\[pytest\]/.test(safeRead(path.join(cwd, "tox.ini"))) ||
+    /\[tool:pytest\]|\[pytest\]/.test(safeRead(path.join(cwd, "setup.cfg"))) ||
+    /\[tool\.pytest/.test(safeRead(path.join(cwd, "pyproject.toml")));
+  if (hasPytest) return { argv: ["pytest"], label: "pytest" };
+  return null;
+}
+
+function safeRead(p) {
+  try {
+    return fs.readFileSync(p, "utf8");
+  } catch {
+    return "";
+  }
+}
+
+// Run the resolved command. Returns {code, output, label} or {enoent:true}.
+// A timeout or signal termination is reported as a non-zero "timed out" result.
+function runTests(resolved) {
+  const opts = { encoding: "utf8", timeout: 120000, maxBuffer: 10 * 1024 * 1024 };
+  let r;
+  if (resolved.shellCmd) {
+    r = spawnSync(resolved.shellCmd, { ...opts, shell: true });
+  } else {
+    r = spawnSync(resolved.argv[0], resolved.argv.slice(1), opts);
+  }
+  const output = `${r.stdout || ""}\n${r.stderr || ""}`;
+  // A timeout or a signal termination is a non-zero "terminated" run (A2).
+  if ((r.error && r.error.code === "ETIMEDOUT") || r.signal) {
+    const why = r.signal ? `terminated: ${r.signal}` : "timed out";
+    return { code: 124, output: `${output}\n[${why}]`, timedOut: true };
+  }
+  // Any other spawn error (ENOENT, EACCES, …) means the command could not run →
+  // can't-verify, not a violation (the caller maps this to exit 2).
+  if (r.error) return { cannotRun: r.error.code || r.error.message };
+  // Shell "command not found" (127) / "not executable" (126) are also can't-run,
+  // not a test failure — a missing --cmd binary must not read as a red suite.
+  if (r.status === 127) return { cannotRun: "command not found (127)" };
+  if (r.status === 126) return { cannotRun: "command not executable (126)" };
+  return { code: r.status == null ? 1 : r.status, output };
+}
+
+function checkRerun(artifactArg, cmdOverride) {
+  let artifactText;
+  try {
+    artifactText = fs.readFileSync(artifactArg, "utf8");
+  } catch {
+    return { usageError: `cannot read artifact: ${artifactArg}` };
+  }
+  const claim = parseTestSummary(artifactText);
+  if (!claim.verdict) return { noClaim: true };
+
+  // Resolve the command: explicit override (shell) or auto-detect (argv).
+  let resolved;
+  let label;
+  if (cmdOverride) {
+    resolved = { shellCmd: cmdOverride };
+    label = cmdOverride;
+  } else {
+    const det = detectTestCommand(process.cwd());
+    if (!det) {
+      return {
+        usageError:
+          "no test command: none detected (need package.json test script or " +
+          "pytest config) — pass --cmd \"<command>\"",
+      };
+    }
+    resolved = { argv: det.argv };
+    label = det.label;
+  }
+
+  // Print the command BEFORE running it (D2): if the run hangs or is killed,
+  // the user still sees what was executed.
+  console.log(`rerun: running \`${label}\` …`);
+  const run = runTests(resolved);
+  if (run.cannotRun) {
+    return { usageError: `could not run test command \`${label}\` (${run.cannotRun})` };
+  }
+  const actual = parseTestSummary(run.output);
+  const actualPass = run.code === 0;
+
+  const violations = [];
+  const claimStr = `${claim.passed != null ? claim.passed + " passed" : "pass"}${
+    claim.failed != null ? ", " + claim.failed + " failed" : ""
+  }`;
+  if (claim.verdict === "pass" && !actualPass) {
+    violations.push(
+      `artifact claims ${claimStr}, but the suite ${
+        run.timedOut ? "timed out" : `failed (exit ${run.code})`
+      }`
+    );
+  } else if (claim.verdict === "fail" && actualPass) {
+    violations.push(
+      `artifact claims ${claimStr}, but the suite passed (exit 0)`
+    );
+  }
+  // Count divergence, when both sides are parseable.
+  if (
+    claim.passed != null &&
+    actual.passed != null &&
+    claim.passed !== actual.passed
+  ) {
+    violations.push(
+      `claimed ${claim.passed} passed, actual run reported ${actual.passed} passed`
+    );
+  }
+  if (
+    claim.failed != null &&
+    actual.failed != null &&
+    claim.failed !== actual.failed
+  ) {
+    violations.push(
+      `claimed ${claim.failed} failed, actual run reported ${actual.failed} failed`
+    );
+  }
+  return { violations, label, actual, actualPass };
+}
+
 // ---------------------------------------------------------------- main
 function main() {
   const argv = process.argv.slice(2);
@@ -261,21 +438,60 @@ function main() {
   }
 
   let artifact = null;
+  let rerunArtifact = null;
+  let cmdOverride = null;
+  let cmdSeen = false;
   let doCitations = false;
   let doTripwires = false;
+  let doRerun = false;
+  let doDetectOnly = false;
+  const takesValue = (v) => v && !v.startsWith("-");
   for (let i = 0; i < argv.length; i++) {
     if (argv[i] === "--citations") {
       doCitations = true;
-      artifact = argv[i + 1] && !argv[i + 1].startsWith("-") ? argv[++i] : null;
+      artifact = takesValue(argv[i + 1]) ? argv[++i] : null;
     } else if (argv[i] === "--tripwires") {
       doTripwires = true;
+    } else if (argv[i] === "--rerun") {
+      doRerun = true;
+      rerunArtifact = takesValue(argv[i + 1]) ? argv[++i] : null;
+    } else if (argv[i] === "--cmd") {
+      cmdSeen = true;
+      cmdOverride = takesValue(argv[i + 1]) ? argv[++i] : null;
+    } else if (argv[i] === "--detect-only") {
+      doDetectOnly = true;
     } else {
       console.error(`Unknown argument: ${argv[i]}\n${USAGE}`);
       return 2;
     }
   }
-  // Default: no mode flag → tripwires (citations require an artifact arg).
+
-  if (!doCitations && !doTripwires) doTripwires = true;
+  // Validate --cmd before any mode runs: it needs a value and only pairs with
+  // --rerun (checked here so --detect-only --cmd is also rejected).
+  if (cmdSeen && cmdOverride == null) {
+    console.error(`--cmd requires a command string\n${USAGE}`);
+    return 2;
+  }
+  if (cmdSeen && !doRerun) {
+    console.error(`--cmd is only valid with --rerun\n${USAGE}`);
+    return 2;
+  }
+
+  // --detect-only: print the detected command (or exit 2 if none). No execution.
+  if (doDetectOnly) {
+    const det = detectTestCommand(process.cwd());
+    if (!det) {
+      console.error(
+        "detect-only: no test command detected (need a package.json test " +
+          'script or pytest config). Pass --cmd "<command>" to override.'
+      );
+      return 2;
+    }
+    console.log(`detected test command: ${det.label}`);
+    return 0;
+  }
+  // Default: no mode flag → tripwires (citations/rerun require an artifact arg).
+  if (!doCitations && !doTripwires && !doRerun) doTripwires = true;
 
   const findings = [];
   let anyViolation = false;
@@ -314,6 +530,30 @@ function main() {
     }
   }
 
+  if (doRerun) {
+    if (!rerunArtifact) {
+      console.error(`--rerun requires an artifact path\n${USAGE}`);
+      return 2;
+    }
+    const res = checkRerun(rerunArtifact, cmdOverride);
+    if (res.usageError) {
+      console.error(res.usageError);
+      return 2;
+    }
+    if (res.noClaim) {
+      findings.push("rerun: no test claim found in artifact — nothing to verify");
+    } else {
+      findings.push(`rerun: ran \`${res.label}\` → ${res.actualPass ? "passed" : "failed"}`);
+      if (res.violations.length) {
+        anyViolation = true;
+        findings.push(`rerun: ${res.violations.length} claim mismatch(es)`);
+        for (const v of res.violations) findings.push(`  ✗ ${v}`);
+      } else {
+        findings.push(`rerun: claim matches the actual run`);
+      }
+    }
+  }
+
   const out = findings.join("\n");
   if (anyViolation) {
     console.error(out);