From 478d6acfa2cc0329c55fa73a61a531fddfb93c2f Mon Sep 17 00:00:00 2001 From: Nick Shirokov Date: Wed, 29 Jul 2026 21:15:40 +0300 Subject: [PATCH] =?UTF-8?q?test(runner):=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5?= =?UTF-8?q?=D1=80=D0=BA=D0=B8=20stdout=20=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=B0=D1=8E=D1=82=20=D0=B8=20=D0=B2=20=D0=BD=D0=B5=D0=B3=D0=B0?= =?UTF-8?q?=D1=82=D0=B8=D0=B2=D0=BD=D1=8B=D1=85=20=D0=BA=D0=B5=D0=B9=D1=81?= =?UTF-8?q?=D0=B0=D1=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit expect.stdoutContains/stdoutNotContains жили в ветке успеха, поэтому у кейса с expectError проверялся только ненулевой код возврата, а строки не смотрелись вовсе. Пятнадцать кейсов (включая xdto-validate, meta-validate, form-validate, meta-remove) были зелёными вхолостую — у facet-conflicts текст навыка успел разойтись с ожиданием. Плюс case-level "cwd": "workDir" — кейсу может понадобиться, чтобы навык стартовал внутри рабочего каталога (фикстура .v8-project.json). Кейсы на доп. аргументы платформы: pass-through 1cv8 и ibcmd, источник из реестра проекта, цепочка epf-build → stub, конфликт ключа, позиционный токен ibcmd, чужой движок, маскирование секрета. Co-Authored-By: Claude Opus 5 (1M context) --- .../cases/db-create/extra-args-conflict.json | 2 +- .../db-create/extra-args-from-project.json | 28 +++++++++++++++---- .../extra-args-ibcmd-passthrough.json | 2 +- .../db-create/extra-args-passthrough-py.json | 12 ++++++++ .../db-create/extra-args-passthrough.json | 1 + .../cases/db-run/extra-args-passthrough.json | 2 +- .../cases/xdto-validate/facet-conflicts.json | 2 +- tests/skills/runner.mjs | 23 +++++++++++---- 8 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 tests/skills/cases/db-create/extra-args-passthrough-py.json diff --git a/tests/skills/cases/db-create/extra-args-conflict.json b/tests/skills/cases/db-create/extra-args-conflict.json index 0ced8ad1..5c897560 100644 --- a/tests/skills/cases/db-create/extra-args-conflict.json +++ b/tests/skills/cases/db-create/extra-args-conflict.json @@ -4,7 +4,7 @@ { "writeFile": { "path": "fake.cmd", "content": "@echo off\r\nexit /b 0\r\n" } } ], "args_extra": ["-V8Path", "{workDir}\\fake.cmd", "-InfoBasePath", "{workDir}\\ib", - "-AdditionalV8Arguments", "/F", "C:\\tmp"], + "-AdditionalV8Arguments", "/F,C:\\tmp"], "expectError": true, "noSnapshot": "негативный кейс: платформа не запускается" } diff --git a/tests/skills/cases/db-create/extra-args-from-project.json b/tests/skills/cases/db-create/extra-args-from-project.json index 5d68ccf1..81eed912 100644 --- a/tests/skills/cases/db-create/extra-args-from-project.json +++ b/tests/skills/cases/db-create/extra-args-from-project.json @@ -1,11 +1,29 @@ { "name": "Доп. аргументы берутся из v8args в .v8-project.json (fake platform)", "preRun": [ - { "writeFile": { "path": "fake.cmd", "content": "@echo off\r\necho ARGS: %*\r\nexit /b 0\r\n" } }, - { "writeFile": { "path": ".v8-project.json", "content": "{\n \"v8path\": \"fake\",\n \"v8args\": [\"/UseHwLicenses+\"],\n \"ibcmdargs\": [\"--verbose\"]\n}\n" } } + { + "writeFile": { + "path": "fake.cmd", + "content": "@echo off\r\necho ARGS: %*\r\nexit /b 0\r\n" + } + }, + { + "writeFile": { + "path": ".v8-project.json", + "content": "{\n \"v8path\": \"fake\",\n \"v8args\": [\"/UseHwLicenses+\"],\n \"ibcmdargs\": [\"--verbose\"]\n}\n" + } + } ], - "args_extra": ["-V8Path", "{workDir}\\fake.cmd", "-InfoBasePath", "{workDir}\\ib"], - "expect": { "stdoutContains": "/DisableStartupDialogs /UseHwLicenses+" }, + "args_extra": [ + "-V8Path", + "{workDir}\\fake.cmd", + "-InfoBasePath", + "{workDir}\\ib" + ], + "expect": { + "stdoutContains": "/DisableStartupDialogs /UseHwLicenses+" + }, "expectError": true, - "noSnapshot": "проверяется командная строка в stdout, файлов на выходе нет" + "noSnapshot": "проверяется командная строка в stdout, файлов на выходе нет", + "cwd": "workDir" } diff --git a/tests/skills/cases/db-create/extra-args-ibcmd-passthrough.json b/tests/skills/cases/db-create/extra-args-ibcmd-passthrough.json index 43f1c372..5e7275ae 100644 --- a/tests/skills/cases/db-create/extra-args-ibcmd-passthrough.json +++ b/tests/skills/cases/db-create/extra-args-ibcmd-passthrough.json @@ -4,7 +4,7 @@ { "writeFile": { "path": "ibcmd.cmd", "content": "@echo off\r\necho ARGS: %*\r\nexit /b 0\r\n" } } ], "args_extra": ["-V8Path", "{workDir}\\ibcmd.cmd", "-InfoBasePath", "{workDir}\\ib", - "-AdditionalIbcmdArguments", "--verbose", "--token=topsecret"], + "-AdditionalIbcmdArguments", "--verbose,--token=topsecret"], "expect": { "stdoutContains": ["ARGS: infobase create", "--verbose --token=topsecret", "--token=***"] }, diff --git a/tests/skills/cases/db-create/extra-args-passthrough-py.json b/tests/skills/cases/db-create/extra-args-passthrough-py.json new file mode 100644 index 00000000..3d5cbb9e --- /dev/null +++ b/tests/skills/cases/db-create/extra-args-passthrough-py.json @@ -0,0 +1,12 @@ +{ + "name": "Доп. аргументы 1cv8 попадают в командную строку, py-порт (fake platform)", + "runtimeOnly": "python", + "preRun": [ + { "writeFile": { "path": "fake.cmd", "content": "@echo off\r\necho ARGS: %*\r\nexit /b 0\r\n" } } + ], + "args_extra": ["-V8Path", "{workDir}\\fake.cmd", "-InfoBasePath", "{workDir}\\ib", + "-AdditionalV8Arguments", "/UseHwLicenses+"], + "expect": { "stdoutContains": "/DisableStartupDialogs /UseHwLicenses+" }, + "expectError": true, + "noSnapshot": "py-порт печатает лог /Out, а не stdout платформы — проверяется строка запуска" +} diff --git a/tests/skills/cases/db-create/extra-args-passthrough.json b/tests/skills/cases/db-create/extra-args-passthrough.json index db186002..48d9e3fc 100644 --- a/tests/skills/cases/db-create/extra-args-passthrough.json +++ b/tests/skills/cases/db-create/extra-args-passthrough.json @@ -5,6 +5,7 @@ ], "args_extra": ["-V8Path", "{workDir}\\fake.cmd", "-InfoBasePath", "{workDir}\\ib", "-AdditionalV8Arguments", "/UseHwLicenses+"], + "runtimeOnly": "powershell", "expect": { "stdoutContains": ["ARGS: CREATEINFOBASE", "/DisableStartupDialogs /UseHwLicenses+"] }, "expectError": true, "noSnapshot": "проверяется командная строка в stdout, файлов на выходе нет" diff --git a/tests/skills/cases/db-run/extra-args-passthrough.json b/tests/skills/cases/db-run/extra-args-passthrough.json index 8f402c79..e9318709 100644 --- a/tests/skills/cases/db-run/extra-args-passthrough.json +++ b/tests/skills/cases/db-run/extra-args-passthrough.json @@ -4,7 +4,7 @@ { "writeFile": { "path": "fake.cmd", "content": "@echo off\r\necho ARGS: %*\r\nexit /b 1\r\n" } } ], "args_extra": ["-V8Path", "{workDir}\\fake.cmd", "-InfoBasePath", "{workDir}\\ib", - "-AdditionalV8Arguments", "/ClearCache", "/UseHwLicenses+"], + "-AdditionalV8Arguments", "/ClearCache,/UseHwLicenses+"], "expect": { "stdoutContains": "/DisableStartupDialogs /ClearCache /UseHwLicenses+" }, "expectError": true, "noSnapshot": "проверяется командная строка в stdout, файлов на выходе нет" diff --git a/tests/skills/cases/xdto-validate/facet-conflicts.json b/tests/skills/cases/xdto-validate/facet-conflicts.json index 95b15286..0e8628e7 100644 --- a/tests/skills/cases/xdto-validate/facet-conflicts.json +++ b/tests/skills/cases/xdto-validate/facet-conflicts.json @@ -8,7 +8,7 @@ "expectError": true, "expect": { "stdoutContains": [ - "length несовместим с minLength/maxLength", + "length задан вместе с minLength/maxLength", "minLength (3) больше maxLength (2)", "fractionDigits (9) больше totalDigits (5)" ] diff --git a/tests/skills/runner.mjs b/tests/skills/runner.mjs index 0a1cf73d..5666a92e 100644 --- a/tests/skills/runner.mjs +++ b/tests/skills/runner.mjs @@ -653,7 +653,7 @@ async function runCaseAsync(testCase, opts) { const { scriptPath, args } = buildArgs(skillConfig, caseData, workDir, inputFile, opts.runtime); let stdout = '', stderr = '', exitCode = 0; try { - const execCwd = skillConfig.cwd === 'workDir' ? workDir : undefined; + const execCwd = (caseData.cwd || skillConfig.cwd) === 'workDir' ? workDir : undefined; stdout = await execSkillAsync(opts.runtime, scriptPath, args, execCwd); } catch (e) { exitCode = e.status ?? 1; @@ -679,6 +679,10 @@ async function runCaseAsync(testCase, opts) { if (!existsSync(join(workDir, f))) errors.push(`Expected file not found: ${f}`); } } + } + // stdout checks apply to negative cases too — a case that says what the failure must + // print was silently checking nothing when they lived in the positive branch only. + { if (caseData.expect?.stdoutContains) { const needles = Array.isArray(caseData.expect.stdoutContains) ? caseData.expect.stdoutContains : [caseData.expect.stdoutContains]; @@ -693,6 +697,8 @@ async function runCaseAsync(testCase, opts) { if (stdout.includes(needle)) errors.push(`stdout unexpectedly contains "${needle}"`); } } + } + if (!caseData.expectError) { if (caseData.expect?.preserves) { const specs = Array.isArray(caseData.expect.preserves) ? caseData.expect.preserves : [caseData.expect.preserves]; @@ -713,7 +719,7 @@ async function runCaseAsync(testCase, opts) { if (errors.length === 0 && caseData.idempotent && !workspace.readOnly) { const before = snapshotWorkDirBytes(workDir); try { - const execCwd = skillConfig.cwd === 'workDir' ? workDir : undefined; + const execCwd = (caseData.cwd || skillConfig.cwd) === 'workDir' ? workDir : undefined; await execSkillAsync(opts.runtime, scriptPath, args, execCwd); } catch (e) { errors.push(`Idempotency rerun failed: exitCode=${e.status}\nstderr: ${(e.stderr || '').substring(0, 300)}`); @@ -835,7 +841,7 @@ function runCase(testCase, opts) { let stdout = '', stderr = '', exitCode = 0; try { - const execCwd = skillConfig.cwd === 'workDir' ? workDir : undefined; + const execCwd = (caseData.cwd || skillConfig.cwd) === 'workDir' ? workDir : undefined; stdout = execSkillRaw(opts.runtime, scriptPath, args, execCwd); } catch (e) { exitCode = e.status ?? 1; @@ -871,8 +877,12 @@ function runCase(testCase, opts) { } } } + } - // expect.stdoutContains / stdoutNotContains (string or array) + // expect.stdoutContains / stdoutNotContains (string or array) — applies to negative + // cases too: a case that says what the failure must print was silently checking + // nothing while these lived in the positive branch only. + { if (caseData.expect?.stdoutContains) { const needles = Array.isArray(caseData.expect.stdoutContains) ? caseData.expect.stdoutContains : [caseData.expect.stdoutContains]; @@ -887,6 +897,9 @@ function runCase(testCase, opts) { if (stdout.includes(needle)) errors.push(`stdout unexpectedly contains "${needle}"`); } } + } + + if (!caseData.expectError) { if (caseData.expect?.preserves) { const specs = Array.isArray(caseData.expect.preserves) ? caseData.expect.preserves : [caseData.expect.preserves]; @@ -908,7 +921,7 @@ function runCase(testCase, opts) { if (errors.length === 0 && caseData.idempotent && !workspace.readOnly) { const before = snapshotWorkDirBytes(workDir); try { - const execCwd = skillConfig.cwd === 'workDir' ? workDir : undefined; + const execCwd = (caseData.cwd || skillConfig.cwd) === 'workDir' ? workDir : undefined; execSkillRaw(opts.runtime, scriptPath, args, execCwd); } catch (e) { errors.push(`Idempotency rerun failed: exitCode=${e.status}\nstderr: ${(e.stderr || '').substring(0, 300)}`);