mirror of
https://github.com/duthaho/claudekit.git
synced 2026-07-22 20:51:01 +03:00
feat: evidence gate — verify citations and fake-green tripwires (#4)
This commit is contained in:
duthaho
@@ -0,0 +1,344 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Test runner for verify-evidence.cjs (the evidence gate) and its fail-open
|
||||
* hook wrapper verify-evidence-hook.cjs. Zero dependencies, no framework:
|
||||
* builds hermetic fixtures in a temp dir, spawns the scripts, asserts exit
|
||||
* codes. Exit 0 when every case passes, 1 otherwise. A missing script or
|
||||
* spawn error counts as that case's FAIL — the run never aborts.
|
||||
*
|
||||
* Mirrors scripts/test-hooks.cjs conventions.
|
||||
*
|
||||
* Usage: node scripts/test-verify-evidence.cjs
|
||||
*/
|
||||
"use strict";
|
||||
|
||||
const { spawnSync } = require("child_process");
|
||||
const fs = require("fs");
|
||||
const os = require("os");
|
||||
const path = require("path");
|
||||
|
||||
const GATE = path.join(__dirname, "verify-evidence.cjs");
|
||||
const HOOK = path.join(__dirname, "verify-evidence-hook.cjs");
|
||||
|
||||
let pass = 0;
|
||||
let fail = 0;
|
||||
const tmpRoots = [];
|
||||
|
||||
function mkTmp() {
|
||||
const d = fs.mkdtempSync(path.join(os.tmpdir(), "vee-"));
|
||||
tmpRoots.push(d);
|
||||
return d;
|
||||
}
|
||||
|
||||
function runGate(args, { cwd, diffFile, env } = {}) {
|
||||
const e = { ...process.env, ...env };
|
||||
if (diffFile) e.VERIFY_EVIDENCE_DIFF_FILE = diffFile;
|
||||
return spawnSync(process.execPath, [GATE, ...args], {
|
||||
cwd: cwd || process.cwd(),
|
||||
env: e,
|
||||
encoding: "utf8",
|
||||
timeout: 10000,
|
||||
});
|
||||
}
|
||||
|
||||
function runHook(stdin, { diffFile } = {}) {
|
||||
const e = { ...process.env };
|
||||
if (diffFile) e.VERIFY_EVIDENCE_DIFF_FILE = diffFile;
|
||||
return spawnSync(process.execPath, [HOOK], {
|
||||
input: stdin,
|
||||
env: e,
|
||||
encoding: "utf8",
|
||||
timeout: 10000,
|
||||
});
|
||||
}
|
||||
|
||||
function check(name, result, expected) {
|
||||
let status = null;
|
||||
let err = "";
|
||||
if (result.error) err = result.error.message;
|
||||
else status = result.status;
|
||||
const ok = !err && status === expected;
|
||||
if (ok) {
|
||||
pass++;
|
||||
console.log(` ✓ ${name} (exit ${status})`);
|
||||
} else {
|
||||
fail++;
|
||||
console.log(` ✗ ${name} — expected exit ${expected}, got ${err || `exit ${status}`}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- helpers to build fixtures ----
|
||||
function citationDir(artifactBody, realFileLines) {
|
||||
const dir = mkTmp();
|
||||
if (realFileLines != null) {
|
||||
fs.writeFileSync(path.join(dir, "real.js"), realFileLines.join("\n") + "\n");
|
||||
}
|
||||
fs.writeFileSync(path.join(dir, "artifact.md"), artifactBody);
|
||||
return dir;
|
||||
}
|
||||
|
||||
function diffFileWith(body) {
|
||||
const dir = mkTmp();
|
||||
const f = path.join(dir, "captured.diff");
|
||||
fs.writeFileSync(f, body);
|
||||
return f;
|
||||
}
|
||||
|
||||
// A minimal unified-diff builder.
|
||||
function diffAdd(filePath, addedLines) {
|
||||
return (
|
||||
`diff --git a/${filePath} b/${filePath}\n` +
|
||||
`index 111..222 100644\n` +
|
||||
`--- a/${filePath}\n` +
|
||||
`+++ b/${filePath}\n` +
|
||||
`@@ -1,1 +1,${addedLines.length + 1} @@\n` +
|
||||
` context\n` +
|
||||
addedLines.map((l) => `+${l}`).join("\n") +
|
||||
"\n"
|
||||
);
|
||||
}
|
||||
|
||||
function diffDelete(filePath) {
|
||||
return (
|
||||
`diff --git a/${filePath} b/${filePath}\n` +
|
||||
`deleted file mode 100644\n` +
|
||||
`index 111..000\n` +
|
||||
`--- a/${filePath}\n` +
|
||||
`+++ /dev/null\n` +
|
||||
`@@ -1,2 +0,0 @@\n` +
|
||||
`-test('x', () => {});\n` +
|
||||
`-test('y', () => {});\n`
|
||||
);
|
||||
}
|
||||
|
||||
function diffRename(fromPath, toPath) {
|
||||
return (
|
||||
`diff --git a/${fromPath} b/${toPath}\n` +
|
||||
`similarity index 100%\n` +
|
||||
`rename from ${fromPath}\n` +
|
||||
`rename to ${toPath}\n`
|
||||
);
|
||||
}
|
||||
|
||||
const diffConcat = (...parts) => parts.join("");
|
||||
|
||||
// ============ CITATION CASES ============
|
||||
console.log("\nverify-evidence.cjs --citations");
|
||||
{
|
||||
const real = ["line one", "line two", "line three"];
|
||||
|
||||
check(
|
||||
"valid single-line citation",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("Root caused in `real.js:2` per the trace.", real),
|
||||
}),
|
||||
0
|
||||
);
|
||||
|
||||
check(
|
||||
"valid range citation",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("See real.js:1-3 for the fix.", real),
|
||||
}),
|
||||
0
|
||||
);
|
||||
|
||||
check(
|
||||
"out-of-range line",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("Broken at real.js:999.", real),
|
||||
}),
|
||||
1
|
||||
);
|
||||
|
||||
check(
|
||||
"missing file",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("See gone.js:1 for detail.", real),
|
||||
}),
|
||||
1
|
||||
);
|
||||
|
||||
check(
|
||||
"absolute path citation is a violation",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("Config at /etc/hosts:1 shows it.", real),
|
||||
}),
|
||||
1
|
||||
);
|
||||
|
||||
check(
|
||||
"prose colons and URLs are not citations",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir(
|
||||
"The ratio was 3:2 and we hit http://example.com:8080 at 12:30.",
|
||||
real
|
||||
),
|
||||
}),
|
||||
0
|
||||
);
|
||||
|
||||
check(
|
||||
"decimal-colon prose (3.5:2) is not a citation",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("Throughput rose 3.5:2 over baseline, and 18.04:30 held.", real),
|
||||
}),
|
||||
0
|
||||
);
|
||||
|
||||
check(
|
||||
"last-line+1 on a newline-terminated file is out of range",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("See real.js:2 for the change.", ["only one line"]),
|
||||
}),
|
||||
1
|
||||
);
|
||||
|
||||
check(
|
||||
"no citations at all",
|
||||
runGate(["--citations", "artifact.md"], {
|
||||
cwd: citationDir("Just prose, nothing to resolve.", real),
|
||||
}),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
// ============ TRIPWIRE CASES ============
|
||||
console.log("\nverify-evidence.cjs --tripwires");
|
||||
{
|
||||
check(
|
||||
"added .skip in a test file",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.test.js", ["it.skip('later', () => {});"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added xit in a spec file",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.spec.js", ["xit('later', () => {});"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added @pytest.mark.skip",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("tests/test_app.py", ["@pytest.mark.skip(reason='x')"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added @unittest.skip",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("tests/test_app.py", ["@unittest.skip('x')"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added TODO in changed code",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.js", ["// TODO: implement for real"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added FIXME in changed code",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.js", ["# FIXME later"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"deleted test file",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffDelete("tests/test_app.py")) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"test renamed to another test path is fine",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffRename("tests/test_old.py", "tests/test_new.py")) }),
|
||||
0
|
||||
);
|
||||
check(
|
||||
"test renamed OUT of the suite is a violation",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffRename("tests/test_app.py", "src/app.py")) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"added bare @skip decorator",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("tests/test_app.py", ["@skip"])) }),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"multi-file diff: clean file then skip in a later file, attributed correctly",
|
||||
runGate(["--tripwires"], {
|
||||
diffFile: diffFileWith(
|
||||
diffConcat(
|
||||
diffAdd("src/util.js", ["const y = 2;"]),
|
||||
diffAdd("src/util.test.js", ["it.skip('later', () => {});"])
|
||||
)
|
||||
),
|
||||
}),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"multi-file diff: deleted non-test then clean adds does not false-flag",
|
||||
runGate(["--tripwires"], {
|
||||
diffFile: diffFileWith(diffConcat(diffDelete("src/legacy.js"), diffAdd("src/new.js", ["const z = 3;"]))),
|
||||
}),
|
||||
0
|
||||
);
|
||||
check(
|
||||
"clean diff of normal code",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffAdd("src/app.js", ["const x = 1;", "return x;"])) }),
|
||||
0
|
||||
);
|
||||
check(
|
||||
"empty diff",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith("") }),
|
||||
0
|
||||
);
|
||||
check(
|
||||
"deleting a NON-test file is fine",
|
||||
runGate(["--tripwires"], { diffFile: diffFileWith(diffDelete("src/legacy.js")) }),
|
||||
0
|
||||
);
|
||||
check(
|
||||
"no git and no diff file -> skipped, not a failure",
|
||||
runGate(["--tripwires"], { cwd: mkTmp() }),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
// ============ COMBINED / NO-FLAG / HELP ============
|
||||
console.log("\nverify-evidence.cjs combined & meta");
|
||||
{
|
||||
const dir = citationDir("Broken at real.js:999.", ["a", "b"]);
|
||||
check(
|
||||
"combined: bad citation + skip tripwire -> exit 1",
|
||||
runGate(["--citations", "artifact.md", "--tripwires"], {
|
||||
cwd: dir,
|
||||
diffFile: diffFileWith(diffAdd("a.test.js", ["it.skip('x', () => {})"])),
|
||||
}),
|
||||
1
|
||||
);
|
||||
check(
|
||||
"no flags runs tripwires (clean diff) -> exit 0",
|
||||
runGate([], { cwd: mkTmp(), diffFile: diffFileWith("") }),
|
||||
0
|
||||
);
|
||||
check("--help exits 0", runGate(["--help"]), 0);
|
||||
}
|
||||
|
||||
// ============ HOOK (fail-open) ============
|
||||
console.log("\nverify-evidence-hook.cjs (always exit 0)");
|
||||
{
|
||||
const stop = JSON.stringify({ hook_event_name: "Stop" });
|
||||
check("valid stop event, clean diff", runHook(stop, { diffFile: diffFileWith("") }), 0);
|
||||
check(
|
||||
"violation present -> hook still exits 0 (advisory)",
|
||||
runHook(stop, { diffFile: diffFileWith(diffAdd("a.test.js", ["it.skip('x', () => {})"])) }),
|
||||
0
|
||||
);
|
||||
check("malformed stdin -> exit 0", runHook("{not json", {}), 0);
|
||||
check("empty stdin -> exit 0", runHook("", {}), 0);
|
||||
}
|
||||
|
||||
// ---- cleanup ----
|
||||
for (const d of tmpRoots) {
|
||||
try {
|
||||
fs.rmSync(d, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n${fail === 0 ? "OK" : "FAIL"} — ${pass} passed, ${fail} failed`);
|
||||
process.exit(fail === 0 ? 0 : 1);
|
||||
Reference in New Issue
Block a user