From ad9264e309bd5e5422c04784372d7841b1e5d604 Mon Sep 17 00:00:00 2001 From: Michael Sitarzewski Date: Sat, 12 Sep 2026 11:57:20 -0500 Subject: [PATCH] fix(convert): track code fences when splitting OpenClaw sections, and pin it with an invariant (#855) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit convert_openclaw() split on `## ` without tracking fenced code blocks, tearing six fenced examples across SOUL.md/AGENTS.md in five agents and leaving dangling fences in each. Implementation from #854 (@guozi-lab): fence helpers in lib.sh used by convert.sh and lint-agents.sh, CommonMark indent aware. Regression test from #853 (@dajiaohuang): every source fenced block must land whole in exactly one output file — verified to fail on the unfixed converter. Reported by @AmineOzil. Closes #849. Supersedes #853 and #854. Co-Authored-By: fruit <200041037+guozi-lab@users.noreply.github.com> Co-Authored-By: Wu Shuwen <108231307+dajiaohuang@users.noreply.github.com> Co-Authored-By: Claude Opus 5 --- scripts/convert-outputs.sha256 | 10 ++--- scripts/convert.sh | 21 +++++++++++ scripts/lib.sh | 28 ++++++++++++++ scripts/lint-agents.sh | 21 +++++++++++ scripts/test-convert-outputs.sh | 65 +++++++++++++++++++++++++++++++++ 5 files changed, 140 insertions(+), 5 deletions(-) diff --git a/scripts/convert-outputs.sha256 b/scripts/convert-outputs.sha256 index 2796a815..6269cbfa 100644 --- a/scripts/convert-outputs.sha256 +++ b/scripts/convert-outputs.sha256 @@ -32,7 +32,7 @@ agent blender-add-on-engineer e5ecd8bfb7e26cfc80af6ee87b374002738d84c11f18a0a3c7 agent blockchain-security-auditor 274ecbce59d30e14b3bed8467aa95d6b4b8e048ebec62db9a5741497820cc754 agent book-co-author 90258d7c0a671bda082309f6db6866ae8e8c3d91e4d6595756ea4ca6d3f838f8 agent bookkeeper-controller 6ca9d331d3679a2891cea5ce5614cbad48492b29fe5055a172264f70dc159cda -agent brand-guardian 57e1bc6418f5e7baa54398930f218501a6398ef1bc9eaf2513baca6f534f8607 +agent brand-guardian dea5745a03b631364669280808f074f954e21a70985fc1d1ea8eddde3b21d2db agent business-strategist f493985599df0171ed24c9f8de888d0c323ce6aaa3f444a2b8461ad1ac153d76 agent carousel-growth-engine c7ca846ed4f0049b733a771d774b10d0ece6dcbfdb026894a25213b1632f7366 agent cartography-designer 058fb8e306bb54ce2eca3a991b4d833281911bc3028ad895e7f05ff6e8c8ec2f @@ -44,7 +44,7 @@ agent china-market-localization-strategist a583d08e874b2702f93bfef8498cac10d1452 agent china-network-engineer 59a437e3a73b60baca0d4ad27d38049626ce5d6048aedcdfa2b6552e3966ce12 agent civil-engineer c0516e459263f598a9924bfec0d588a477d8c70077214d33daadcc20cd99349f agent clinical-evidence-agent 718b61dd3392b242e4f19832ab8f5bd63aec7f0cfe45e18d190fae96994517ab -agent cloud-security-architect bcb4a2946891364563aefaa5576a2833aaf1910b7efa17731832b91b28afda07 +agent cloud-security-architect 08a35ec1bc40e37ad02ed6251a5dcad56e676e5179735fdd0f353a78a0326226 agent cms-developer 8e3e31fdc7cb34781f579251df294b012b6b119780f4b2c1cbf36c2fcdc0464c agent code-reviewer 71c91ed253267724c89a43a914fae4992bb58e4776e5d39b7bbe713d12201e24 agent codebase-archaeologist db6736949c483d325937695b335536ecaad070d237c8d67cc15eb6c841dceb79 @@ -102,7 +102,7 @@ agent gis-analyst 6aad2e21fab62a0b5c8afa6e9a1d1f147ab40e1560532a833f8ff534c6ddd9 agent gis-qa-engineer 08d424435b10b3cd95be785ae281cedce374575599d2e07e53dca9429ba6c361 agent git-workflow-master cbea1fc502bcc242bed933183c3017334dd0d94235433d71432b2c2f23e6caaf agent global-podcast-strategist 825862d052b361eb9fa9b9558c1bc46c230f9268235f97c889a8035a59518be7 -agent godot-gameplay-scripter 7c5f43d180334f9df710fb3e25bb10d0bd798ad3b43c4c11f8c59cc4ea3562f2 +agent godot-gameplay-scripter 9a16be8310a999ad7f3e6a1072dadce7075208ec28e86e5fabafc905b1458714 agent godot-multiplayer-engineer 6968b96da873411b6ff30a5f71322482116dc02205ebe071942465ccf473e3a9 agent godot-shader-developer eed41026d50e5a5b099a67f9eb1fe9747f509345b4d44d528e3e2beb72902148 agent government-digital-presales-consultant 86a743df9a5d29305d85a578071ef47d7542489f57438ae7770daecbd2d7302d @@ -115,10 +115,10 @@ agent historian d0c6ab0da820c2ecd44e2cd55a49811afb5b676065060837b78644ca551c910a agent hospitality-guest-services 8250743b1b7b8f7b9dbd733bf1a7199601681f4f5e9e89e48431c6e4df522d43 agent hr-onboarding 74768b5133b57873419525819600daae93f6550f45d8de161c18dfde5a4a1461 agent identity-access-engineer cd21e9fdae6133e93ad53b011e0fbbc5f8266b2e29c7a4dba53f30de80c5077a -agent identity-graph-operator 1fb3600479d4afa48d3cbc6e30358246bf0a320a59e4e9818b367d116c846ae6 +agent identity-graph-operator 369e0b2458f8b560ca155e87b13360da50d4362c4e3ec8a67bedde7b4e983a6d agent image-prompt-engineer fb37bda0f7e509964029cabfa8e629edbb53e946043f686ec8bfe532439771fc agent incident-responder 923cb920f4649b48e41f3074a2727b441a48c7b2978b4b9f27cde39d648b07ba -agent incident-response-commander b5fca7e26de3f7c3eee73b2b3db4051d6a2efe0843ac8a1a29d6aee6a1b14454 +agent incident-response-commander b9febd98828f0d3f655a56ab827c4c14560f3260495e0ccfdbd4eb7d2129bb9d agent inclusive-visuals-specialist ddb31d99e461efa49301bb24685148295a77881622fe783c536c5270a84e9e7a agent infrastructure-maintainer af51f8c47f62b87d22dc2356bc9904e7126e03420614f968566dfac113ea2c7a agent instagram-curator 4b0d5e941c793cb354bbab3d2498b7e673e42f8486d9b8ccb019a20bdb0852f2 diff --git a/scripts/convert.sh b/scripts/convert.sh index d557a0c7..e379d858 100755 --- a/scripts/convert.sh +++ b/scripts/convert.sh @@ -328,8 +328,29 @@ convert_openclaw() { local current_target="agents" # default bucket local current_section="" + # While fence_marker is set, ## lines are code content, not section + # boundaries (issue #849). See lib.sh fence_open_p / fence_closes_p. + local fence_marker="" fence_len=0 fence_indent=0 while IFS= read -r line; do + if [[ -n "$fence_marker" ]]; then + current_section+="$line"$'\n' + if fence_closes_p "$line" "$fence_marker" "$fence_len" "$fence_indent"; then + fence_marker="" + fence_len=0 + fence_indent=0 + fi + continue + fi + + if fence_open_p "$line"; then + fence_marker="${BASH_REMATCH[2]:0:1}" + fence_len=${#BASH_REMATCH[2]} + fence_indent=${#BASH_REMATCH[1]} + current_section+="$line"$'\n' + continue + fi + # Detect ## headers (with or without emoji prefixes) if [[ "$line" =~ ^##[[:space:]] ]]; then # Flush previous section diff --git a/scripts/lib.sh b/scripts/lib.sh index 1f507cb3..146c45b5 100755 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -63,6 +63,34 @@ is_agent_file() { [[ -f "$1" ]] && [[ "$(head -1 "$1")" == "---" ]] } +# --------------------------------------------------------------------------- +# 1b. Markdown fenced-code-block helpers (issue #849) +# --------------------------------------------------------------------------- + +# fence_open_p — 0 if opens a fence (3+ ` or ~, 0–3 leading +# spaces); sets BASH_REMATCH[1]=indent, [2]=marker run. Read those directly, +# not via $(), so the per-line convert/lint loops stay subshell-free. Else 1. +fence_open_p() { + local line="$1" + local re='^( {0,3})(`{3,}|~{3,})' + [[ "$line" =~ $re ]] +} + +# fence_closes_p — 0 if +# closes the open fence (same char, run len >= open, indent <= open); 1 +# otherwise, including non-fence lines (callers need not pre-classify). +fence_closes_p() { + local line="$1" open_marker="$2" open_len="$3" open_indent="$4" + local re='^( {0,3})(`{3,}|~{3,})' + [[ "$line" =~ $re ]] || return 1 + local close_indent=${#BASH_REMATCH[1]} + local close_run="${BASH_REMATCH[2]}" + [[ "${close_run:0:1}" == "$open_marker" ]] || return 1 + (( ${#close_run} >= open_len )) || return 1 + (( close_indent <= open_indent )) || return 1 + return 0 +} + # --------------------------------------------------------------------------- # 2. set -e-safe primitives (absorbs #505 — no more `(( x++ )) || true`) # --------------------------------------------------------------------------- diff --git a/scripts/lint-agents.sh b/scripts/lint-agents.sh index 9f6815a1..93ab657e 100755 --- a/scripts/lint-agents.sh +++ b/scripts/lint-agents.sh @@ -10,6 +10,10 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +. "$SCRIPT_DIR/lib.sh" + # Keep in sync with AGENT_DIRS in scripts/convert.sh AGENT_DIRS=( academic @@ -123,7 +127,24 @@ lint_file() { local soul_headers=0 local agents_headers=0 + local fence_marker="" fence_len=0 fence_indent=0 while IFS= read -r line; do + # Skip fenced code blocks so ## doc-comment lines (e.g. GDScript `##`) + # and in-fence markdown headers aren't miscounted (issue #849). + if [[ -n "$fence_marker" ]]; then + if fence_closes_p "$line" "$fence_marker" "$fence_len" "$fence_indent"; then + fence_marker="" + fence_len=0 + fence_indent=0 + fi + continue + fi + if fence_open_p "$line"; then + fence_marker="${BASH_REMATCH[2]:0:1}" + fence_len=${#BASH_REMATCH[2]} + fence_indent=${#BASH_REMATCH[1]} + continue + fi if [[ "$line" =~ ^##[[:space:]] ]]; then local header_lower header_lower=$(printf '%s' "$line" | tr '[:upper:]' '[:lower:]') diff --git a/scripts/test-convert-outputs.sh b/scripts/test-convert-outputs.sh index 8f6c03ec..4b5676d4 100755 --- a/scripts/test-convert-outputs.sh +++ b/scripts/test-convert-outputs.sh @@ -288,6 +288,71 @@ for tool in TOOLS: report(tool, bad_parse, bad_trip, "parse and round-trip" if fmt in ("yaml-fm", "toml") else "parse, carry their slug, and have their prose file") +# --- Layer A (split integrity): a source fenced block must survive whole ------- +# openclaw is the one tool that splits a single agent body across two files, at +# `## ` headings: SOUL.md (persona) / AGENTS.md (operations). A heading inside a +# fenced code block is block content, not a boundary. Splitting on one cuts the +# block in half and leaves each file holding a dangling fence, which renders as +# broken markdown for every user of that integration (#849). So every fenced +# block in a source must land intact in exactly one of the two files. +SPLIT_FENCE = re.compile(r"^(`{3,}|~{3,})") + +def body_lines(text): + """Mirror lib.sh's get_body, including `$(...)`'s trailing-newline strip.""" + out, fm = [], 0 + for line in text.split("\n"): + if line == "---": + fm += 1 + continue + if fm >= 2: + out.append(line) + while out and out[-1] == "": + out.pop() + return out + +def fence_blocks(lines): + """Inclusive (opener, closer) index pairs; closer = last line if unterminated.""" + res, marker, mlen, start = [], "", 0, None + for i, line in enumerate(lines): + m = SPLIT_FENCE.match(line) + if not m: + continue + tok = m.group(1) + if not marker: + marker, mlen, start = tok[0], len(tok), i + elif tok[0] == marker and len(tok) >= mlen: + res.append((start, i)); marker, mlen, start = "", 0, None + if marker and start is not None: + res.append((start, len(lines) - 1)) + return res + +def has_run(hay, needle): + n = len(needle) + return n > 0 and n <= len(hay) and any(hay[i:i+n] == needle for i in range(len(hay) - n + 1)) + +split_bad = 0 +for slug, (_d, _n, path) in sorted(src.items()): + sfile = os.path.join(OUT, "openclaw", slug, "SOUL.md") + afile = os.path.join(OUT, "openclaw", slug, "AGENTS.md") + if not (os.path.isfile(sfile) and os.path.isfile(afile)): + continue + body = body_lines(open(os.path.join(R, path), encoding="utf-8").read()) + bl = fence_blocks(body) + if not bl: + continue + outs = [open(sfile, encoding="utf-8").read().split("\n"), + open(afile, encoding="utf-8").read().split("\n")] + for a, b in bl: + if not any(has_run(o, body[a:b + 1]) for o in outs): + split_bad += 1 + if split_bad <= 3: + bad(f"openclaw: {slug} source fenced block at lines {a+1}-{b+1} is torn across " + f"SOUL.md/AGENTS.md — a `## ` heading inside the fence was treated as a section boundary") +if split_bad: + if split_bad > 3: bad(f"openclaw: ...and {split_bad-3} more torn fenced blocks") +else: + ok(f"openclaw: all {N} agents keep every source fenced block whole in one output file") + # --- Layer A (app-facing): every SOURCE frontmatter strict-parsed above ------- for m in src_bad[:5]: bad(m) if len(src_bad) > 5: bad(f"...and {len(src_bad)-5} more source frontmatter problems")