mirror of
https://github.com/duthaho/claudekit.git
synced 2026-07-22 12:41:00 +03:00
feat: evidence gate — verify citations and fake-green tripwires (#4)
This commit is contained in:
@@ -14,3 +14,5 @@ jobs:
|
||||
run: node scripts/validate-plugin.cjs
|
||||
- name: Check frontmatter token budgets
|
||||
run: node scripts/token-report.cjs --check
|
||||
- name: Test the evidence gate
|
||||
run: node scripts/test-verify-evidence.cjs
|
||||
|
||||
@@ -12,7 +12,9 @@ docs website (`website/`, Astro Starlight). The only repo-level check is
|
||||
- Plugin: nothing to build. Lint before committing: `node scripts/validate-plugin.cjs`
|
||||
— validates every SKILL.md against the 8-section anatomy and plugin/marketplace
|
||||
version sync; `node scripts/token-report.cjs --check` — enforces token budgets
|
||||
on skill/agent frontmatter (both run in CI, `.github/workflows/validate.yml`).
|
||||
on skill/agent frontmatter; `node scripts/test-verify-evidence.cjs` — exercises
|
||||
the evidence gate (`scripts/verify-evidence.cjs`: citation resolution +
|
||||
fake-green tripwires). All three run in CI, `.github/workflows/validate.yml`.
|
||||
- Test by installing locally:
|
||||
`/plugin marketplace add <path-or-repo>` → `/plugin install claudekit` → `/claudekit:init`
|
||||
- Website (`cd website/`): `npm install`, `npm run dev`, `npm run build`,
|
||||
|
||||
@@ -24,6 +24,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
skill/agent frontmatter with named budgets, enforced in CI via `--check` (#2)
|
||||
- CI workflow `.github/workflows/validate.yml` running the anatomy and
|
||||
token-budget checks on pushes to `main` and PRs (#1, #2)
|
||||
- `scripts/verify-evidence.cjs` — evidence gate that mechanically checks an
|
||||
agent's claimed evidence: resolves `file:line` citations against real
|
||||
paths/ranges (`--citations`) and scans `git diff HEAD` for fake-green
|
||||
tampering — deleted test files, added test skips, new TODO/FIXME
|
||||
(`--tripwires`); loud gate (exit 1 violations, 2 usage/crash). Deliberately
|
||||
does not attempt deep test-tamper analysis
|
||||
- `scripts/verify-evidence-hook.cjs` — fail-open Stop/PostToolUse wrapper that
|
||||
surfaces tripwire findings as advisory, never blocking
|
||||
- `scripts/test-verify-evidence.cjs` — zero-dependency fixture runner for the
|
||||
evidence gate (26 cases), added to CI
|
||||
|
||||
## [4.0.0] - 2026-05-07
|
||||
|
||||
|
||||
@@ -161,15 +161,22 @@ In practice, devs skip steps for trivial work. The chains show the full discipli
|
||||
## Development
|
||||
|
||||
CI (`.github/workflows/validate.yml`) lints every skill against the 8-section
|
||||
anatomy above, checks `plugin.json`/`marketplace.json` version sync, and
|
||||
enforces token budgets on the always-loaded skill/agent frontmatter (the
|
||||
"no agent-bloat" claim, measured). Run locally before committing:
|
||||
anatomy above, checks `plugin.json`/`marketplace.json` version sync, enforces
|
||||
token budgets on the always-loaded skill/agent frontmatter (the "no agent-bloat"
|
||||
claim, measured), and exercises the evidence gate. Run locally before committing:
|
||||
|
||||
```
|
||||
node scripts/validate-plugin.cjs
|
||||
node scripts/token-report.cjs --check
|
||||
node scripts/test-verify-evidence.cjs
|
||||
```
|
||||
|
||||
`scripts/verify-evidence.cjs` is the evidence gate itself — dogfooding the
|
||||
"every claim has evidence" philosophy against the agent's own output. It
|
||||
resolves `file:line` citations in an artifact (`--citations <file>`) and scans
|
||||
`git diff HEAD` for fake-green tampering — deleted test files, added test skips,
|
||||
new TODO/FIXME (`--tripwires`) — exiting non-zero when a claim can't be verified.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Claude Code 1.0+
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Test runner for verify-evidence.cjs (the evidence gate) and its fail-open
|
||||
* hook wrapper verify-evidence-hook.cjs. Zero dependencies, no framework:
|
||||
* builds hermetic fixtures in a temp dir, spawns the scripts, asserts exit
|
||||
* codes. Exit 0 when every case passes, 1 otherwise. A missing script or
|
||||
* spawn error counts as that case's FAIL — the run never aborts.
|
||||
*
|
||||
* Mirrors scripts/test-hooks.cjs conventions.
|
||||
*
|
||||
* Usage: node scripts/test-verify-evidence.cjs
|
||||
*/
|
||||
"use strict";
|
||||
|
||||
const { spawnSync } = require("child_process");
|
||||
const fs = require("fs");
|
||||
const os = require("os");
|
||||
const path = require("path");
|
||||
|
||||
const GATE = path.join(__dirname, "verify-evidence.cjs");
|
||||
const HOOK = path.join(__dirname, "verify-evidence-hook.cjs");
|
||||
|
||||
let pass = 0;
|
||||
let fail = 0;
|
||||
const tmpRoots = [];
|
||||
|
||||
function mkTmp() {
|
||||
const d = fs.mkdtempSync(path.join(os.tmpdir(), "vee-"));
|
||||
tmpRoots.push(d);
|
||||
return d;
|
||||
}
|
||||
|
||||
function runGate(args, { cwd, diffFile, env } = {}) {
|
||||
const e = { ...process.env, ...env };
|
||||
if (diffFile) e.VERIFY_EVIDENCE_DIFF_FILE = diffFile;
|
||||
return spawnSync(process.execPath, [GATE, ...args], {
|
||||
cwd: cwd || process.cwd(),
|
||||
env: e,
|
||||
encoding: "utf8",
|
||||
timeout: 10000,
|
||||
});
|
||||
}
|
||||
|
||||
function runHook(stdin, { diffFile } = {}) {
|
||||
const e = { ...process.env };
|
||||
if (diffFile) e.VERIFY_EVIDENCE_DIFF_FILE = diffFile;
|
||||
return spawnSync(process.execPath, [HOOK], {
|
||||
input: stdin,
|
||||
env: e,
|
||||
encoding: "utf8",
|
||||
timeout: 10000,
|
||||
});
|
||||
}
|
||||
|
||||
function check(name, result, expected) {
|
||||
let status = null;
|
||||
let err = "";
|
||||
if (result.error) err = result.error.message;
|
||||
else status = result.status;
|
||||
const ok = !err && status === expected;
|
||||
if (ok) {
|
||||
pass++;
|
||||
console.log(` ✓ ${name} (exit ${status})`);
|
||||
} else {
|
||||
fail++;
|
||||
console.log(` ✗ ${name} — expected exit ${expected}, got ${err || `exit ${status}`}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- helpers to build fixtures ----
|
||||
function citationDir(artifactBody, realFileLines) {
|
||||
const dir = mkTmp();
|
||||
if (realFileLines != null) {
|
||||
fs.writeFileSync(path.join(dir, "real.js"), realFileLines.join("\n") + "\n");
|
||||
}
|
||||
fs.writeFileSync(path.join(dir, "artifact.md"), artifactBody);
|
||||
return dir;
|
||||
}
|
||||
|
||||
function diffFileWith(body) {
|
||||
const dir = mkTmp();
|
||||
const f = path.join(dir, "captured.diff");
|
||||
fs.writeFileSync(f, body);
|
||||
return f;
|
||||
}
|
||||
|
||||
// A minimal unified-diff builder.
|
||||
function diffAdd(filePath, addedLines) {
|
||||
return (
|
||||
`diff --git a/${filePath} b/${filePath}\n` +
|
||||
`index 111..222 100644\n` +
|
||||
`--- a/${filePath}\n` +
|
||||
`+++ b/${filePath}\n` +
|
||||
`@@ -1,1 +1,${addedLines.length + 1} @@\n` +
|
||||
` context\n` +
|
||||
addedLines.map((l) => `+${l}`).join("\n") +
|
||||
"\n"
|
||||
);
|
||||
}
|
||||
|
||||
function diffDelete(filePath) {
|
||||
return (
|
||||
`diff --git a/${filePath} b/${filePath}\n` +
|
||||
`deleted file mode 100644\n` +
|
||||
`index 111..000\n` +
|
||||
`--- a/${filePath}\n` +
|
||||
`+++ /dev/null\n` +
|
||||
`@@ -1,2 +0,0 @@\n` +
|
||||
`-test('x', () => {});\n` +
|
||||
`-test('y', () => {});\n`
|
||||
);
|
||||
}
|
||||
|
||||
function diffRename(fromPath, toPath) {
|
||||
return (
|
||||
`diff --git a/${fromPath} b/${toPath}\n` +
|
||||
`similarity index 100%\n` +
|
||||
`rename from ${fromPath}\n` +
|
||||
`rename to ${toPath}\n`
|
||||
);
|
||||
}
|
||||
|
||||
const diffConcat = (...parts) => parts.join("");
|
||||
|
||||
// ============ CITATION CASES ============
|
||||
console.log("\nverify-evidence.cjs --citations");
|
||||
{
|
||||
const real = ["line one", "line two", "line three"];
|
||||
|
||||
check(
|
||||
"valid single-line citation",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("Root caused in `real.js:2` per the trace.", real),
|
||||
}),
|
||||
0
|
||||
);
|
||||
|
||||
check(
|
||||
"valid range citation",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("See real.js:1-3 for the fix.", real),
|
||||
}),
|
||||
0
|
||||
);
|
||||
|
||||
check(
|
||||
"out-of-range line",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("Broken at real.js:999.", real),
|
||||
}),
|
||||
1
|
||||
);
|
||||
|
||||
check(
|
||||
"missing file",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("See gone.js:1 for detail.", real),
|
||||
}),
|
||||
1
|
||||
);
|
||||
|
||||
check(
|
||||
"absolute path citation is a violation",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("Config at /etc/hosts:1 shows it.", real),
|
||||
}),
|
||||
1
|
||||
);
|
||||
|
||||
check(
|
||||
"prose colons and URLs are not citations",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir(
|
||||
"The ratio was 3:2 and we hit http://example.com:8080 at 12:30.",
|
||||
real
|
||||
),
|
||||
}),
|
||||
0
|
||||
);
|
||||
|
||||
check(
|
||||
"decimal-colon prose (3.5:2) is not a citation",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("Throughput rose 3.5:2 over baseline, and 18.04:30 held.", real),
|
||||
}),
|
||||
0
|
||||
);
|
||||
|
||||
check(
|
||||
"last-line+1 on a newline-terminated file is out of range",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("See real.js:2 for the change.", ["only one line"]),
|
||||
}),
|
||||
1
|
||||
);
|
||||
|
||||
check(
|
||||
"no citations at all",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("Just prose, nothing to resolve.", real),
|
||||
}),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
// ============ TRIPWIRE CASES ============
|
||||
console.log("\nverify-evidence.cjs --tripwires");
|
||||
{
|
||||
check(
|
||||
"added .skip in a test file",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.test.js", ["it.skip('later', () => {});"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added xit in a spec file",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.spec.js", ["xit('later', () => {});"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added @pytest.mark.skip",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("tests/test_app.py", ["@pytest.mark.skip(reason='x')"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added @unittest.skip",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("tests/test_app.py", ["@unittest.skip('x')"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added TODO in changed code",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.js", ["// TODO: implement for real"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added FIXME in changed code",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.js", ["# FIXME later"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"deleted test file",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffDelete("tests/test_app.py")) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"test renamed to another test path is fine",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffRename("tests/test_old.py", "tests/test_new.py")) }),
|
||||
0
|
||||
);
|
||||
check(
|
||||
"test renamed OUT of the suite is a violation",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffRename("tests/test_app.py", "src/app.py")) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added bare @skip decorator",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("tests/test_app.py", ["@skip"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"multi-file diff: clean file then skip in a later file, attributed correctly",
|
||||
runGate(["--tripwires"], {
|
||||
diffFile: diffFileWith(
|
||||
diffConcat(
|
||||
diffAdd("src/util.js", ["const y = 2;"]),
|
||||
diffAdd("src/util.test.js", ["it.skip('later', () => {});"])
|
||||
)
|
||||
),
|
||||
}),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"multi-file diff: deleted non-test then clean adds does not false-flag",
|
||||
runGate(["--tripwires"], {
|
||||
diffFile: diffFileWith(diffConcat(diffDelete("src/legacy.js"), diffAdd("src/new.js", ["const z = 3;"]))),
|
||||
}),
|
||||
0
|
||||
);
|
||||
check(
|
||||
"clean diff of normal code",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.js", ["const x = 1;", "return x;"])) }),
|
||||
0
|
||||
);
|
||||
check(
|
||||
"empty diff",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith("") }),
|
||||
0
|
||||
);
|
||||
check(
|
||||
"deleting a NON-test file is fine",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffDelete("src/legacy.js")) }),
|
||||
0
|
||||
);
|
||||
check(
|
||||
"no git and no diff file -> skipped, not a failure",
|
||||
runGate(["--tripwires"], { cwd: mkTmp() }),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
// ============ COMBINED / NO-FLAG / HELP ============
|
||||
console.log("\nverify-evidence.cjs combined & meta");
|
||||
{
|
||||
const dir = citationDir("Broken at real.js:999.", ["a", "b"]);
|
||||
check(
|
||||
"combined: bad citation + skip tripwire -> exit 1",
|
||||
runGate(["--citations", "artifact.md", "--tripwires"], {
|
||||
cwd: dir,
|
||||
diffFile: diffFileWith(diffAdd("a.test.js", ["it.skip('x', () => {})"])),
|
||||
}),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"no flags runs tripwires (clean diff) -> exit 0",
|
||||
runGate([], { cwd: mkTmp(), diffFile: diffFileWith("") }),
|
||||
0
|
||||
);
|
||||
check("--help exits 0", runGate(["--help"]), 0);
|
||||
}
|
||||
|
||||
// ============ HOOK (fail-open) ============
|
||||
console.log("\nverify-evidence-hook.cjs (always exit 0)");
|
||||
{
|
||||
const stop = JSON.stringify({ hook_event_name: "Stop" });
|
||||
check("valid stop event, clean diff", runHook(stop, { diffFile: diffFileWith("") }), 0);
|
||||
check(
|
||||
"violation present -> hook still exits 0 (advisory)",
|
||||
runHook(stop, { diffFile: diffFileWith(diffAdd("a.test.js", ["it.skip('x', () => {})"])) }),
|
||||
0
|
||||
);
|
||||
check("malformed stdin -> exit 0", runHook("{not json", {}), 0);
|
||||
check("empty stdin -> exit 0", runHook("", {}), 0);
|
||||
}
|
||||
|
||||
// ---- cleanup ----
|
||||
for (const d of tmpRoots) {
|
||||
try {
|
||||
fs.rmSync(d, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n${fail === 0 ? "OK" : "FAIL"} — ${pass} passed, ${fail} failed`);
|
||||
process.exit(fail === 0 ? 0 : 1);
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Fail-open hook wrapper around the evidence gate (scripts/verify-evidence.cjs).
|
||||
* Intended for a Stop / PostToolUse hook: after the agent claims work is done,
|
||||
* run the tripwire scan and surface any fake-green findings as ADVISORY output
|
||||
* — it never blocks and never exits non-zero on its own errors.
|
||||
*
|
||||
* Fails open (exit 0 always), per the repo's hook convention
|
||||
* (see scripts/auto-format.cjs, scripts/detect-secrets.cjs). The loud,
|
||||
* blocking enforcement lives in verify-evidence.cjs run as a CI gate; this
|
||||
* wrapper is the low-friction in-session nudge.
|
||||
*/
|
||||
"use strict";
|
||||
|
||||
const path = require("path");
|
||||
const { spawnSync } = require("child_process");
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
// Drain stdin (hook payload); content is not needed for the diff scan.
|
||||
let data = "";
|
||||
for await (const chunk of process.stdin) data += chunk;
|
||||
try {
|
||||
JSON.parse(data);
|
||||
} catch {
|
||||
// Malformed / empty payload is fine — this hook fires opportunistically.
|
||||
}
|
||||
|
||||
const gate = path.join(__dirname, "verify-evidence.cjs");
|
||||
const r = spawnSync(process.execPath, [gate, "--tripwires"], {
|
||||
encoding: "utf8",
|
||||
timeout: 10000,
|
||||
});
|
||||
|
||||
// Exit 1 from the gate means tripwires were found — surface them, advisory.
|
||||
if (r.status === 1 && (r.stderr || r.stdout)) {
|
||||
console.error("[evidence] fake-green tripwires detected (advisory):");
|
||||
console.error(r.stderr || r.stdout);
|
||||
}
|
||||
} catch {
|
||||
// Fail open — a hook bug must never stall or block the session.
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,334 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Evidence gate: mechanically checks that an AI agent's *claimed* evidence is
|
||||
* real, rather than trusting the transcript. Two modes:
|
||||
*
|
||||
* --citations <artifact.md> Resolve `file:line` / `file:start-end` citations
|
||||
* in an evidence artifact; fail if a path is
|
||||
* missing, absolute/outside the repo, or the line
|
||||
* is out of range.
|
||||
* --tripwires Scan `git diff HEAD` for fake-green tampering:
|
||||
* deleted test files, newly skipped tests, and new
|
||||
* TODO/FIXME left in changed code.
|
||||
*
|
||||
* With no mode flag, runs --tripwires (citations need an artifact argument).
|
||||
*
|
||||
* This is a GATE, not a convenience hook: it fails loud.
|
||||
* exit 0 = clean exit 1 = violations found exit 2 = usage / internal error
|
||||
* (Exit 2 is kept distinct from 1 so a crash can never masquerade as a
|
||||
* "violation", per scripts/test-verify-evidence.cjs.)
|
||||
*
|
||||
* Precision-first by design: it would rather miss an oddly-shaped citation or an
|
||||
* unusually-named test than cry wolf — a gate that cries wolf gets disabled. It
|
||||
* deliberately does NOT attempt deep test-tamper analysis (assertion-count
|
||||
* deltas, mutation proofs); that is out of scope for this slice.
|
||||
*
|
||||
* Zero dependencies. Usage: node scripts/verify-evidence.cjs [--citations <f>] [--tripwires]
|
||||
*/
|
||||
"use strict";
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { spawnSync } = require("child_process");
|
||||
|
||||
const USAGE =
|
||||
"Usage: node scripts/verify-evidence.cjs [--citations <artifact.md>] [--tripwires]";
|
||||
|
||||
// ---------------------------------------------------------------- citations
|
||||
// A citation is `path:line` or `path:start-end`, backtick-optional. To avoid
|
||||
// false positives from prose ("ratio 3:2", "12:30") and URLs, the path token
|
||||
// must contain a "." or "/", URLs are stripped first, and a match preceded by a
|
||||
// word char or ":" is rejected.
|
||||
const CITATION_RE =
|
||||
/(?<![\w:])(\/?[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*):(\d+)(?:-(\d+))?(?![\d-])/g;
|
||||
|
||||
function findCitations(text) {
|
||||
const stripped = text.replace(/\bhttps?:\/\/\S+/gi, " ");
|
||||
const out = [];
|
||||
let m;
|
||||
CITATION_RE.lastIndex = 0;
|
||||
while ((m = CITATION_RE.exec(stripped)) !== null) {
|
||||
const p = m[1];
|
||||
// Path-like only: contains a "/" or a letter-led extension (".js", ".py").
|
||||
// A dotted number like "3.5" (in prose "ratio 3.5:2") is NOT a path.
|
||||
if (!p.includes("/") && !/\.[A-Za-z]/.test(p)) continue;
|
||||
out.push({
|
||||
raw: m[0],
|
||||
filePath: p,
|
||||
start: parseInt(m[2], 10),
|
||||
end: m[3] ? parseInt(m[3], 10) : parseInt(m[2], 10),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Repo root so citations resolve the same way regardless of the cwd the gate
|
||||
// is run from; falls back to cwd when not in a git work tree (e.g. tests).
|
||||
function repoRoot() {
|
||||
const r = spawnSync("git", ["rev-parse", "--show-toplevel"], {
|
||||
encoding: "utf8",
|
||||
timeout: 10000,
|
||||
});
|
||||
if (!r.error && r.status === 0 && r.stdout.trim()) {
|
||||
try {
|
||||
return fs.realpathSync(r.stdout.trim());
|
||||
} catch {
|
||||
/* fall through */
|
||||
}
|
||||
}
|
||||
return fs.realpathSync(process.cwd());
|
||||
}
|
||||
|
||||
function checkCitations(artifactArg) {
|
||||
const violations = [];
|
||||
let artifactText;
|
||||
try {
|
||||
artifactText = fs.readFileSync(artifactArg, "utf8");
|
||||
} catch {
|
||||
return { usageError: `cannot read artifact: ${artifactArg}` };
|
||||
}
|
||||
const base = repoRoot();
|
||||
const citations = findCitations(artifactText);
|
||||
|
||||
for (const c of citations) {
|
||||
if (path.isAbsolute(c.filePath)) {
|
||||
violations.push(`${c.raw} — absolute path, cannot verify against repo`);
|
||||
continue;
|
||||
}
|
||||
const full = path.resolve(base, c.filePath);
|
||||
if (!fs.existsSync(full)) {
|
||||
violations.push(`${c.raw} — cited file does not exist`);
|
||||
continue;
|
||||
}
|
||||
let realFull;
|
||||
try {
|
||||
realFull = fs.realpathSync(full);
|
||||
} catch {
|
||||
violations.push(`${c.raw} — cited file does not exist`);
|
||||
continue;
|
||||
}
|
||||
if (realFull !== base && !realFull.startsWith(base + path.sep)) {
|
||||
violations.push(`${c.raw} — resolves outside the repository`);
|
||||
continue;
|
||||
}
|
||||
let stat;
|
||||
try {
|
||||
stat = fs.statSync(realFull);
|
||||
} catch {
|
||||
violations.push(`${c.raw} — cited file does not exist`);
|
||||
continue;
|
||||
}
|
||||
if (!stat.isFile()) {
|
||||
violations.push(`${c.raw} — citation does not point at a file`);
|
||||
continue;
|
||||
}
|
||||
const body = fs.readFileSync(realFull, "utf8");
|
||||
// Count real lines: a single trailing newline is a terminator, not a line.
|
||||
const lineCount = body === "" ? 0 : body.replace(/\n$/, "").split("\n").length;
|
||||
if (c.start < 1 || c.end < c.start || c.end > lineCount) {
|
||||
violations.push(
|
||||
`${c.raw} — line out of range (file has ${lineCount} lines)`
|
||||
);
|
||||
}
|
||||
}
|
||||
return { violations, checked: citations.length };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- tripwires
|
||||
function isTestFile(p) {
|
||||
const f = p.toLowerCase();
|
||||
return (
|
||||
/(^|\/)tests?\//.test(f) ||
|
||||
/(^|\/)__tests__\//.test(f) ||
|
||||
/\.(test|spec)\.[a-z0-9]+$/.test(f) ||
|
||||
/(^|\/)test_[^/]*\.py$/.test(f) ||
|
||||
/_test\.py$/.test(f) ||
|
||||
// "test"/"spec" as a separator-bounded token in the basename — catches
|
||||
// e.g. app.test.js, foo_spec.rb, but not latest.js / inspector.js.
|
||||
/(^|[\/._-])(test|spec)([\/._-]|$)/.test(f)
|
||||
);
|
||||
}
|
||||
|
||||
const SKIP_PATTERNS = [
|
||||
{ name: "skipped test (.skip)", re: /\.skip\s*\(/ },
|
||||
{ name: "skipped test (xit/xdescribe/xtest)", re: /\bx(it|describe|test)\s*\(/ },
|
||||
// Matches @skip, @pytest.mark.skip, @pytest.mark.skipif, @unittest.skip,
|
||||
// @unittest.skipUnless, @unittest.skipIf.
|
||||
{ name: "skipped test (@skip decorator)", re: /@(?:pytest\.mark\.|unittest\.)?skip/ },
|
||||
];
|
||||
const MARKER_RE = /\b(TODO|FIXME)\b/;
|
||||
|
||||
function getDiff() {
|
||||
const override = process.env.VERIFY_EVIDENCE_DIFF_FILE;
|
||||
if (override) {
|
||||
try {
|
||||
return { text: fs.readFileSync(override, "utf8") };
|
||||
} catch {
|
||||
return { unavailable: `cannot read VERIFY_EVIDENCE_DIFF_FILE: ${override}` };
|
||||
}
|
||||
}
|
||||
const r = spawnSync("git", ["diff", "HEAD"], { encoding: "utf8", timeout: 10000 });
|
||||
if (r.error || r.status !== 0) {
|
||||
return { unavailable: "git diff HEAD unavailable (not a git repo?)" };
|
||||
}
|
||||
return { text: r.stdout || "" };
|
||||
}
|
||||
|
||||
function checkTripwires() {
|
||||
const diff = getDiff();
|
||||
if (diff.unavailable) return { skipped: diff.unavailable };
|
||||
|
||||
const violations = [];
|
||||
const lines = diff.text.split("\n");
|
||||
let aPath = null;
|
||||
let bPath = null;
|
||||
let curFile = null;
|
||||
let newLineNo = 0;
|
||||
let renameFrom = null;
|
||||
|
||||
const stripPrefix = (s) =>
|
||||
s === "/dev/null" ? "/dev/null" : s.replace(/^[ab]\//, "");
|
||||
|
||||
for (const line of lines) {
|
||||
// A new file section resets all per-file state, so location and deletion
|
||||
// attribution never leak across files in a multi-file diff.
|
||||
if (line.startsWith("diff --git ")) {
|
||||
aPath = null;
|
||||
bPath = null;
|
||||
curFile = null;
|
||||
newLineNo = 0;
|
||||
renameFrom = null;
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("rename from ")) {
|
||||
renameFrom = line.slice("rename from ".length).trim();
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("rename to ")) {
|
||||
const renameTo = line.slice("rename to ".length).trim();
|
||||
// Renaming a test file to a non-test path removes it from the suite.
|
||||
if (renameFrom && isTestFile(renameFrom) && !isTestFile(renameTo)) {
|
||||
violations.push(`test file renamed out of the suite: ${renameFrom} → ${renameTo}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("--- ")) {
|
||||
aPath = stripPrefix(line.slice(4).trim());
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("+++ ")) {
|
||||
bPath = stripPrefix(line.slice(4).trim());
|
||||
curFile = bPath === "/dev/null" ? aPath : bPath;
|
||||
// A file whose new side is /dev/null was deleted.
|
||||
if (bPath === "/dev/null" && aPath && isTestFile(aPath)) {
|
||||
violations.push(`deleted test file: ${aPath}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line);
|
||||
if (hunk) {
|
||||
newLineNo = parseInt(hunk[1], 10);
|
||||
continue;
|
||||
}
|
||||
// Added content line (not the "+++ " file header, handled above).
|
||||
if (line.startsWith("+")) {
|
||||
const content = line.slice(1);
|
||||
const loc = curFile ? `${curFile}:${newLineNo}` : "(unknown)";
|
||||
for (const s of SKIP_PATTERNS) {
|
||||
if (s.re.test(content)) violations.push(`${s.name} added at ${loc}`);
|
||||
}
|
||||
if (MARKER_RE.test(content)) {
|
||||
violations.push(`TODO/FIXME added at ${loc}: ${content.trim()}`);
|
||||
}
|
||||
newLineNo++;
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith(" ")) {
|
||||
newLineNo++;
|
||||
continue;
|
||||
}
|
||||
// "-" lines and metadata: no new-side advance
|
||||
}
|
||||
return { violations };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- main
|
||||
function main() {
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv.includes("--help") || argv.includes("-h")) {
|
||||
console.log(USAGE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
let artifact = null;
|
||||
let doCitations = false;
|
||||
let doTripwires = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === "--citations") {
|
||||
doCitations = true;
|
||||
artifact = argv[i + 1] && !argv[i + 1].startsWith("-") ? argv[++i] : null;
|
||||
} else if (argv[i] === "--tripwires") {
|
||||
doTripwires = true;
|
||||
} else {
|
||||
console.error(`Unknown argument: ${argv[i]}\n${USAGE}`);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
// Default: no mode flag → tripwires (citations require an artifact arg).
|
||||
if (!doCitations && !doTripwires) doTripwires = true;
|
||||
|
||||
const findings = [];
|
||||
let anyViolation = false;
|
||||
|
||||
if (doCitations) {
|
||||
if (!artifact) {
|
||||
console.error(`--citations requires an artifact path\n${USAGE}`);
|
||||
return 2;
|
||||
}
|
||||
const res = checkCitations(artifact);
|
||||
if (res.usageError) {
|
||||
console.error(res.usageError);
|
||||
return 2;
|
||||
}
|
||||
if (res.violations.length) {
|
||||
anyViolation = true;
|
||||
findings.push(
|
||||
`citations: ${res.violations.length} unresolved of ${res.checked} checked`
|
||||
);
|
||||
for (const v of res.violations) findings.push(` ✗ ${v}`);
|
||||
} else {
|
||||
findings.push(`citations: ${res.checked} checked, all resolve`);
|
||||
}
|
||||
}
|
||||
|
||||
if (doTripwires) {
|
||||
const res = checkTripwires();
|
||||
if (res.skipped) {
|
||||
findings.push(`tripwires: skipped — ${res.skipped}`);
|
||||
} else if (res.violations.length) {
|
||||
anyViolation = true;
|
||||
findings.push(`tripwires: ${res.violations.length} found`);
|
||||
for (const v of res.violations) findings.push(` ✗ ${v}`);
|
||||
} else {
|
||||
findings.push(`tripwires: none`);
|
||||
}
|
||||
}
|
||||
|
||||
const out = findings.join("\n");
|
||||
if (anyViolation) {
|
||||
console.error(out);
|
||||
console.error("\nFAIL — evidence gate found unverifiable claims.");
|
||||
return 1;
|
||||
}
|
||||
console.log(out);
|
||||
console.log("\nOK — evidence gate passed.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
process.exit(main());
|
||||
} catch (err) {
|
||||
// Internal error is exit 2, never 1 — a crash must not read as a violation.
|
||||
console.error(`verify-evidence: internal error: ${err && err.message}`);
|
||||
process.exit(2);
|
||||
}
|
||||
Reference in New Issue
Block a user