creator/cc-1c-skills
1
0
Fork 0
mirror of https://github.com/Nikolay-Shirokov/cc-1c-skills.git synced 2026-07-31 08:47:45 +03:00
Code Issues Packages Projects Releases Wiki Activity

test(runner): проверки stdout работают и в негативных кейсах

Browse Source 
expect.stdoutContains/stdoutNotContains жили в ветке успеха, поэтому у
кейса с expectError проверялся только ненулевой код возврата, а строки
не смотрелись вовсе. Пятнадцать кейсов (включая xdto-validate,
meta-validate, form-validate, meta-remove) были зелёными вхолостую —
у facet-conflicts текст навыка успел разойтись с ожиданием.

Плюс case-level "cwd": "workDir" — кейсу может понадобиться, чтобы
навык стартовал внутри рабочего каталога (фикстура .v8-project.json).

Кейсы на доп. аргументы платформы: pass-through 1cv8 и ibcmd, источник
из реестра проекта, цепочка epf-build → stub, конфликт ключа,
позиционный токен ibcmd, чужой движок, маскирование секрета.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Shirokov
2026-07-29 21:15:40 +03:00
co-authored by Claude Opus 5
parent 761e7b8613
commit 478d6acfa2
8 changed files with 58 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/skills/cases/db-create/extra-args-conflict.json
+1 -1
View File
@@ -4,7 +4,7 @@
{ "writeFile": { "path": "fake.cmd", "content": "@echo off\r\nexit /b 0\r\n" } }
],
"args_extra": ["-V8Path", "{workDir}\\fake.cmd", "-InfoBasePath", "{workDir}\\ib",
"-AdditionalV8Arguments", "/F", "C:\\tmp"],
"-AdditionalV8Arguments", "/F,C:\\tmp"],
"expectError": true,
"noSnapshot": "негативный кейс: платформа не запускается"
}
tests/skills/cases/db-create/extra-args-from-project.json
+23 -5
View File
@@ -1,11 +1,29 @@
{
"name": "Доп. аргументы берутся из v8args в .v8-project.json (fake platform)",
"preRun": [
{ "writeFile": { "path": "fake.cmd", "content": "@echo off\r\necho ARGS: %*\r\nexit /b 0\r\n" } },
{ "writeFile": { "path": ".v8-project.json", "content": "{\n \"v8path\": \"fake\",\n \"v8args\": [\"/UseHwLicenses+\"],\n \"ibcmdargs\": [\"--verbose\"]\n}\n" } }
{
"writeFile": {
"path": "fake.cmd",
"content": "@echo off\r\necho ARGS: %*\r\nexit /b 0\r\n"
}
},
{
"writeFile": {
"path": ".v8-project.json",
"content": "{\n \"v8path\": \"fake\",\n \"v8args\": [\"/UseHwLicenses+\"],\n \"ibcmdargs\": [\"--verbose\"]\n}\n"
}
}
],
"args_extra": ["-V8Path", "{workDir}\\fake.cmd", "-InfoBasePath", "{workDir}\\ib"],
"expect": { "stdoutContains": "/DisableStartupDialogs /UseHwLicenses+" },
"args_extra": [
"-V8Path",
"{workDir}\\fake.cmd",
"-InfoBasePath",
"{workDir}\\ib"
],
"expect": {
"stdoutContains": "/DisableStartupDialogs /UseHwLicenses+"
},
"expectError": true,
"noSnapshot": "проверяется командная строка в stdout, файлов на выходе нет"
"noSnapshot": "проверяется командная строка в stdout, файлов на выходе нет",
"cwd": "workDir"
}
tests/skills/cases/db-create/extra-args-ibcmd-passthrough.json
+1 -1
View File
@@ -4,7 +4,7 @@
{ "writeFile": { "path": "ibcmd.cmd", "content": "@echo off\r\necho ARGS: %*\r\nexit /b 0\r\n" } }
],
"args_extra": ["-V8Path", "{workDir}\\ibcmd.cmd", "-InfoBasePath", "{workDir}\\ib",
"-AdditionalIbcmdArguments", "--verbose", "--token=topsecret"],
"-AdditionalIbcmdArguments", "--verbose,--token=topsecret"],
"expect": {
"stdoutContains": ["ARGS: infobase create", "--verbose --token=topsecret", "--token=***"]
},
tests/skills/cases/db-create/extra-args-passthrough-py.json
+12
View File
@@ -0,0 +1,12 @@
{
"name": "Доп. аргументы 1cv8 попадают в командную строку, py-порт (fake platform)",
"runtimeOnly": "python",
"preRun": [
{ "writeFile": { "path": "fake.cmd", "content": "@echo off\r\necho ARGS: %*\r\nexit /b 0\r\n" } }
],
"args_extra": ["-V8Path", "{workDir}\\fake.cmd", "-InfoBasePath", "{workDir}\\ib",
"-AdditionalV8Arguments", "/UseHwLicenses+"],
"expect": { "stdoutContains": "/DisableStartupDialogs /UseHwLicenses+" },
"expectError": true,
"noSnapshot": "py-порт печатает лог /Out, а не stdout платформы — проверяется строка запуска"
}
tests/skills/cases/db-create/extra-args-passthrough.json
+1
View File
@@ -5,6 +5,7 @@
],
"args_extra": ["-V8Path", "{workDir}\\fake.cmd", "-InfoBasePath", "{workDir}\\ib",
"-AdditionalV8Arguments", "/UseHwLicenses+"],
"runtimeOnly": "powershell",
"expect": { "stdoutContains": ["ARGS: CREATEINFOBASE", "/DisableStartupDialogs /UseHwLicenses+"] },
"expectError": true,
"noSnapshot": "проверяется командная строка в stdout, файлов на выходе нет"
tests/skills/cases/db-run/extra-args-passthrough.json
+1 -1
View File
@@ -4,7 +4,7 @@
{ "writeFile": { "path": "fake.cmd", "content": "@echo off\r\necho ARGS: %*\r\nexit /b 1\r\n" } }
],
"args_extra": ["-V8Path", "{workDir}\\fake.cmd", "-InfoBasePath", "{workDir}\\ib",
"-AdditionalV8Arguments", "/ClearCache", "/UseHwLicenses+"],
"-AdditionalV8Arguments", "/ClearCache,/UseHwLicenses+"],
"expect": { "stdoutContains": "/DisableStartupDialogs /ClearCache /UseHwLicenses+" },
"expectError": true,
"noSnapshot": "проверяется командная строка в stdout, файлов на выходе нет"
tests/skills/cases/xdto-validate/facet-conflicts.json
+1 -1
View File
@@ -8,7 +8,7 @@
"expectError": true,
"expect": {
"stdoutContains": [
"length несовместим с minLength/maxLength",
"length задан вместе с minLength/maxLength",
"minLength (3) больше maxLength (2)",
"fractionDigits (9) больше totalDigits (5)"
]
tests/skills/runner.mjs
+18 -5
View File
@@ -653,7 +653,7 @@ async function runCaseAsync(testCase, opts) {
const { scriptPath, args } = buildArgs(skillConfig, caseData, workDir, inputFile, opts.runtime);
let stdout = '', stderr = '', exitCode = 0;
try {
const execCwd = skillConfig.cwd === 'workDir' ? workDir : undefined;
const execCwd = (caseData.cwd || skillConfig.cwd) === 'workDir' ? workDir : undefined;
stdout = await execSkillAsync(opts.runtime, scriptPath, args, execCwd);
} catch (e) {
exitCode = e.status ?? 1;
@@ -679,6 +679,10 @@ async function runCaseAsync(testCase, opts) {
if (!existsSync(join(workDir, f))) errors.push(`Expected file not found: ${f}`);
}
}
}
// stdout checks apply to negative cases too — a case that says what the failure must
// print was silently checking nothing when they lived in the positive branch only.
{
if (caseData.expect?.stdoutContains) {
const needles = Array.isArray(caseData.expect.stdoutContains)
? caseData.expect.stdoutContains : [caseData.expect.stdoutContains];
@@ -693,6 +697,8 @@ async function runCaseAsync(testCase, opts) {
if (stdout.includes(needle)) errors.push(`stdout unexpectedly contains "${needle}"`);
}
}
}
if (!caseData.expectError) {
if (caseData.expect?.preserves) {
const specs = Array.isArray(caseData.expect.preserves)
? caseData.expect.preserves : [caseData.expect.preserves];
@@ -713,7 +719,7 @@ async function runCaseAsync(testCase, opts) {
if (errors.length === 0 && caseData.idempotent && !workspace.readOnly) {
const before = snapshotWorkDirBytes(workDir);
try {
const execCwd = skillConfig.cwd === 'workDir' ? workDir : undefined;
const execCwd = (caseData.cwd || skillConfig.cwd) === 'workDir' ? workDir : undefined;
await execSkillAsync(opts.runtime, scriptPath, args, execCwd);
} catch (e) {
errors.push(`Idempotency rerun failed: exitCode=${e.status}\nstderr: ${(e.stderr || '').substring(0, 300)}`);
@@ -835,7 +841,7 @@ function runCase(testCase, opts) {
let stdout = '', stderr = '', exitCode = 0;
try {
const execCwd = skillConfig.cwd === 'workDir' ? workDir : undefined;
const execCwd = (caseData.cwd || skillConfig.cwd) === 'workDir' ? workDir : undefined;
stdout = execSkillRaw(opts.runtime, scriptPath, args, execCwd);
} catch (e) {
exitCode = e.status ?? 1;
@@ -871,8 +877,12 @@ function runCase(testCase, opts) {
}
}
}
}
// expect.stdoutContains / stdoutNotContains (string or array)
// expect.stdoutContains / stdoutNotContains (string or array) — applies to negative
// cases too: a case that says what the failure must print was silently checking
// nothing while these lived in the positive branch only.
{
if (caseData.expect?.stdoutContains) {
const needles = Array.isArray(caseData.expect.stdoutContains)
? caseData.expect.stdoutContains : [caseData.expect.stdoutContains];
@@ -887,6 +897,9 @@ function runCase(testCase, opts) {
if (stdout.includes(needle)) errors.push(`stdout unexpectedly contains "${needle}"`);
}
}
}
if (!caseData.expectError) {
if (caseData.expect?.preserves) {
const specs = Array.isArray(caseData.expect.preserves)
? caseData.expect.preserves : [caseData.expect.preserves];
@@ -908,7 +921,7 @@ function runCase(testCase, opts) {
if (errors.length === 0 && caseData.idempotent && !workspace.readOnly) {
const before = snapshotWorkDirBytes(workDir);
try {
const execCwd = skillConfig.cwd === 'workDir' ? workDir : undefined;
const execCwd = (caseData.cwd || skillConfig.cwd) === 'workDir' ? workDir : undefined;
execSkillRaw(opts.runtime, scriptPath, args, execCwd);
} catch (e) {
errors.push(`Idempotency rerun failed: exitCode=${e.status}\nstderr: ${(e.stderr || '').substring(0, 300)}`);