fix: replace three hand-rolled YAML parsers with a shared PyYAML loader

index.json shipped 604 of 817 descriptions truncated to their first line.
The cause was the inline regex parser in update-index.yml: it reconstructed
multi-line descriptions only for the YAML block-scalar styles ('>' and '|')
and silently dropped continuation lines for every other style.

A census of the corpus explains the blast radius:

  block scalar   (description: >-)    43
  single-quoted multiline            278
  plain unquoted multiline           496
  single-line                          0

So 774 of 817 skills (94.7%) used a style the parser mishandled. Commit
d56fc0a7 had fixed only the 43 block-scalar files, and CONTRIBUTING.md
recommends that one working style, which is why it stayed hidden.

- add tools/skill_frontmatter.py, the single PyYAML-backed loader
- add tools/generate-index.py so generation is testable outside CI, with
  a --check mode for use as a gate
- delete the hand-rolled parsers from validate-skill.py (98 lines) and
  validate-agentskills.py, routing both through the shared loader
- implement the reserved-word check that agentskills-skill.schema.json
  names validate-agentskills.py as the enforcement point for

Verified by a differential harness against yaml.safe_load ground truth:
index-vs-source mismatches 606 -> 0.

.github/workflows/update-index.yml

@@ -18,72 +18,15 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      # Generation lives in tools/generate-index.py so it is testable outside CI
+      # and shares one PyYAML-backed frontmatter parser with the validators.
+      # The previous inline regex parser silently truncated 604/817 descriptions
+      # to their first line for every YAML scalar style except '>'/'|'.
       - name: Regenerate index.json
-        run: |
-          python3 << 'EOF'
-          import os, json, re
-          from datetime import datetime, timezone
-
-          skills_dir = "skills"
-          skills = []
-
-          for skill_name in sorted(os.listdir(skills_dir)):
-              skill_md = os.path.join(skills_dir, skill_name, "SKILL.md")
-              if not os.path.isfile(skill_md):
-                  continue
-              with open(skill_md, "r", encoding="utf-8") as f:
-                  content = f.read()
-              fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
-              description = ""
-              if fm_match:
-                  fm = fm_match.group(1)
-                  dm = re.search(r"^description:[ \t]*(.*)$", fm, re.MULTILINE)
-                  if dm:
-                      first = dm.group(1).strip()
-                      if first[:1] in (">", "|"):
-                          # YAML block scalar: gather the following more-indented lines
-                          buf = []
-                          for ln in fm[dm.end():].split("\n"):
-                              if ln.strip() == "":
-                                  buf.append("")
-                              elif re.match(r"^[ \t]+\S", ln):
-                                  buf.append(ln.strip())
-                              else:
-                                  break
-                          if first.startswith(">"):  # folded: blank line = break, else join w/ space
-                              paras, cur = [], []
-                              for b in buf:
-                                  if b == "":
-                                      if cur: paras.append(" ".join(cur)); cur = []
-                                  else:
-                                      cur.append(b)
-                              if cur: paras.append(" ".join(cur))
-                              description = " ".join(paras).strip()
-                          else:  # literal
-                              description = " ".join(b for b in buf if b).strip()
-                      else:
-                          description = first.strip('"').strip("'")
-              skills.append({
-                  "name": skill_name,
-                  "description": description,
-                  "domain": "cybersecurity",
-                  "path": f"skills/{skill_name}"
-              })
-
-          index = {
-              "version": "1.1.0",
-              "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
-              "repository": "https://github.com/mukul975/Anthropic-Cybersecurity-Skills",
-              "domain": "cybersecurity",
-              "total_skills": len(skills),
-              "skills": skills
-          }
-
-          with open("index.json", "w", encoding="utf-8") as f:
-              json.dump(index, f, separators=(',', ':'))
-
-          print(f"Updated index.json: {len(skills)} skills")
-          EOF
+        run: python3 tools/generate-index.py
 
       - name: Sync skill count into README and marketplace
         run: |

tools/generate-index.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Generate index.json from the SKILL.md files under skills/.
+
+Previously this logic lived as an inline heredoc inside
+.github/workflows/update-index.yml with a hand-rolled regex YAML parser that
+truncated 604 of 817 descriptions. It lives here now so it is testable outside
+CI and shares one PyYAML-backed parser with every other tool.
+
+Usage:
+    python tools/generate-index.py            # write index.json
+    python tools/generate-index.py --check    # verify index.json is current (CI)
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from skill_frontmatter import description_of, iter_skill_dirs, load_frontmatter, FrontmatterError
+
+INDEX_VERSION = "1.1.0"
+REPOSITORY = "https://github.com/mukul975/Anthropic-Cybersecurity-Skills"
+DEFAULT_DOMAIN = "cybersecurity"
+
+
+def build_index(skills_dir: str = "skills") -> tuple[dict, list[str]]:
+    """Build the index payload. Returns (index, errors)."""
+    skills = []
+    errors = []
+
+    for slug, skill_dir in iter_skill_dirs(skills_dir):
+        try:
+            frontmatter = load_frontmatter(os.path.join(skill_dir, "SKILL.md"))
+        except FrontmatterError as exc:
+            errors.append(f"{slug}: {exc}")
+            continue
+
+        description = description_of(frontmatter)
+        if not description:
+            errors.append(f"{slug}: empty description")
+
+        skills.append({
+            "name": slug,
+            "description": description,
+            "domain": frontmatter.get("domain") or DEFAULT_DOMAIN,
+            "path": f"{skills_dir}/{slug}",
+        })
+
+    index = {
+        "version": INDEX_VERSION,
+        "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "repository": REPOSITORY,
+        "domain": DEFAULT_DOMAIN,
+        "total_skills": len(skills),
+        "skills": skills,
+    }
+    return index, errors
+
+
+def _comparable(index: dict) -> str:
+    """Serialize an index ignoring generated_at, so --check tolerates a re-run."""
+    return json.dumps({k: v for k, v in index.items() if k != "generated_at"}, sort_keys=True)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--check", action="store_true",
+                        help="verify index.json matches the skills tree; do not write")
+    parser.add_argument("--skills-dir", default="skills")
+    parser.add_argument("--out", default="index.json")
+    args = parser.parse_args()
+
+    if not os.path.isdir(args.skills_dir):
+        print(f"ERROR: '{args.skills_dir}' not found. Run from the repository root.")
+        return 1
+
+    index, errors = build_index(args.skills_dir)
+
+    for error in errors:
+        print(f"ERROR {error}")
+    if errors:
+        print(f"\n{len(errors)} skill(s) could not be indexed.")
+        return 1
+
+    if args.check:
+        if not os.path.isfile(args.out):
+            print(f"ERROR: {args.out} is missing. Run: python tools/generate-index.py")
+            return 1
+        with open(args.out, encoding="utf-8") as handle:
+            current = json.load(handle)
+        if _comparable(current) != _comparable(index):
+            print(f"ERROR: {args.out} is out of date. Run: python tools/generate-index.py")
+            return 1
+        print(f"OK: {args.out} is up to date ({index['total_skills']} skills)")
+        return 0
+
+    with open(args.out, "w", encoding="utf-8") as handle:
+        json.dump(index, handle, separators=(",", ":"))
+
+    print(f"Updated {args.out}: {index['total_skills']} skills")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/skill_frontmatter.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Single source of truth for reading SKILL.md YAML frontmatter.
+
+Every tool in this repository MUST read frontmatter through this module.
+
+Why this exists
+---------------
+This repo previously carried three independent hand-rolled "YAML-ish" parsers
+(the index generator, validate-skill.py, validate-agentskills.py). Each handled
+a different subset of YAML scalar styles, and all of them silently truncated
+multi-line descriptions to their first line.
+
+A census of the 817 skills shows why that was fatal:
+
+    block scalar   (description: >-)    43
+    single-quoted multiline            278
+    plain unquoted multiline           496
+    single-line                          0
+
+Only the 43 block-scalar files parsed correctly; 774 (94.7%) used a style the
+hand-rolled parsers mishandled, and 604 descriptions shipped truncated in
+index.json with no error and no warning.
+
+PyYAML handles every scalar style, quoting form and escape correctly. Do not
+reintroduce a regex-based frontmatter parser -- CI greps for that.
+"""
+from __future__ import annotations
+
+import os
+import re
+from typing import Dict, Iterator, Tuple
+
+import yaml
+
+# Frontmatter is the block between the opening '---' and the next '---' that
+# sits alone on its own line. Tolerates CRLF and a leading UTF-8 BOM.
+_FRONTMATTER_RE = re.compile(r"\A?---[ \t]*\r?\n(.*?)\r?\n---[ \t]*(?:\r?\n|\Z)", re.DOTALL)
+
+BACKUP_SUFFIX = ".bak"
+
+
+class FrontmatterError(ValueError):
+    """Raised when a SKILL.md has missing or unparseable frontmatter."""
+
+
+def extract_block(text: str) -> str:
+    """Return the raw YAML frontmatter block from a SKILL.md's text."""
+    match = _FRONTMATTER_RE.match(text)
+    if not match:
+        raise FrontmatterError("no YAML frontmatter block (file must start with '---')")
+    return match.group(1)
+
+
+def parse(text: str) -> dict:
+    """Parse a SKILL.md's full text into a frontmatter dict."""
+    try:
+        data = yaml.safe_load(extract_block(text))
+    except yaml.YAMLError as exc:
+        raise FrontmatterError(f"invalid YAML in frontmatter: {exc}") from exc
+
+    if data is None:
+        return {}
+    if not isinstance(data, dict):
+        raise FrontmatterError(f"frontmatter must be a mapping, got {type(data).__name__}")
+    return data
+
+
+def load_frontmatter(skill_md_path: str) -> dict:
+    """Read one SKILL.md and return its frontmatter as a dict."""
+    try:
+        with open(skill_md_path, encoding="utf-8") as handle:
+            text = handle.read()
+    except UnicodeDecodeError as exc:
+        raise FrontmatterError(f"not valid UTF-8: {exc}") from exc
+    return parse(text)
+
+
+def description_of(frontmatter: dict) -> str:
+    """Return the description as a single normalized line.
+
+    YAML preserves the newlines of a literal ('|') scalar and folds a folded
+    ('>') one; collapsing whitespace here gives every style the same shape,
+    which is what index.json and the linters want to compare.
+    """
+    return " ".join(str(frontmatter.get("description", "")).split())
+
+
+def iter_skill_dirs(skills_dir: str = "skills") -> Iterator[Tuple[str, str]]:
+    """Yield (slug, skill_dir) for every real skill, in sorted order.
+
+    Skips '*.bak' backup directories and any directory lacking a SKILL.md.
+    """
+    for slug in sorted(os.listdir(skills_dir)):
+        if slug.endswith(BACKUP_SUFFIX):
+            continue
+        skill_dir = os.path.join(skills_dir, slug)
+        if not os.path.isdir(skill_dir):
+            continue
+        if not os.path.isfile(os.path.join(skill_dir, "SKILL.md")):
+            continue
+        yield slug, skill_dir
+
+
+def load_all(skills_dir: str = "skills") -> Tuple[Dict[str, dict], Dict[str, str]]:
+    """Load frontmatter for every skill.
+
+    Returns (frontmatter_by_slug, errors_by_slug). Callers decide whether a
+    parse failure is fatal; nothing is silently dropped.
+    """
+    loaded: Dict[str, dict] = {}
+    errors: Dict[str, str] = {}
+
+    for slug, skill_dir in iter_skill_dirs(skills_dir):
+        try:
+            loaded[slug] = load_frontmatter(os.path.join(skill_dir, "SKILL.md"))
+        except FrontmatterError as exc:
+            errors[slug] = str(exc)
+
+    return loaded, errors

tools/validate-agentskills.py

@@ -13,62 +13,37 @@ Usage:
 import os, re, sys, json, glob
 from collections import Counter
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from skill_frontmatter import description_of, load_frontmatter, FrontmatterError
+
 REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 ALLOWED = {"name", "description", "license", "compatibility", "metadata", "allowed-tools"}
 NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$")
 
-def top_level_keys_and_scalars(fm):
-    """Minimal YAML: top-level keys (col 0) + scalar values for name/description."""
-    keys = []
-    scalars = {}
-    lines = fm.split("\n")
-    for i, line in enumerate(lines):
-        m = re.match(r"^([A-Za-z0-9_-]+):(.*)$", line)
-        if not m:
-            continue
-        key, rest = m.group(1), m.group(2)
-        keys.append(key)
-        val = rest.strip()
-        if val and val[0] in "|>":  # block scalar -> gather following indented lines
-            buf = []
-            for nxt in lines[i + 1:]:
-                if re.match(r"^\s+\S", nxt):
-                    buf.append(nxt.strip())
-                elif nxt.strip() == "":
-                    buf.append("")
-                else:
-                    break
-            val = " ".join(x for x in buf if x != "").strip()
-        elif not val:
-            # could be a folded plain scalar wrapped onto continuation lines
-            buf = []
-            for nxt in lines[i + 1:]:
-                if re.match(r"^\s+-\s", nxt) or re.match(r"^[A-Za-z0-9_-]+:", nxt):
-                    break
-                if re.match(r"^\s+\S", nxt):
-                    buf.append(nxt.strip())
-                else:
-                    break
-            val = " ".join(buf).strip()
-        scalars[key] = val.strip().strip("\"'")
-    return keys, scalars
+# The agentskills.io standard forbids reserved vendor words in a skill name.
+# tools/agentskills-skill.schema.json names this script as the enforcement
+# point, but the check was never actually implemented until now.
+RESERVED_NAME_WORDS = ("anthropic", "claude")
+
 
 def validate(path):
     slug = os.path.basename(os.path.dirname(path))
-    text = open(path, encoding="utf-8").read()
-    m = re.match(r"^---\n(.*?)\n---", text, re.DOTALL)
     problems = []
-    if not m:
-        return slug, ["no YAML frontmatter block"], []
-    fm = m.group(1)
-    keys, scalars = top_level_keys_and_scalars(fm)
+
+    try:
+        fm = load_frontmatter(path)
+    except FrontmatterError as exc:
+        return slug, [str(exc)], []
+
+    keys = list(fm.keys())
 
     if "name" not in keys:
         problems.append("missing required key: name")
     if "description" not in keys:
         problems.append("missing required key: description")
 
-    name = scalars.get("name", "")
+    name = str(fm.get("name", "") or "")
     if name:
         if not NAME_RE.match(name):
             problems.append(f"name not lowercase-kebab-case: {name!r}")
@@ -76,19 +51,25 @@ def validate(path):
             problems.append(f"name length {len(name)} out of 1..64")
         if name != slug:
             problems.append(f"name {name!r} != directory {slug!r}")
+        for reserved in RESERVED_NAME_WORDS:
+            if reserved in name.lower():
+                problems.append(f"name contains reserved word {reserved!r}: {name!r}")
 
-    desc = scalars.get("description", "")
+    desc = description_of(fm)
     if desc:
         if not (1 <= len(desc) <= 1024):
             problems.append(f"description length {len(desc)} out of 1..1024")
     elif "description" in keys:
         problems.append("description empty")
 
-    # Ignore YAML block-scalar indicators (`key: >`, `key: >-`, `key: |`, ...);
-    # only genuine `<...>`/`>` content in values is an injection concern.
-    fm_no_ind = re.sub(r":[ \t]*[|>][+-]?[ \t]*(?=\n|$)", ":", fm)
-    if "<" in fm_no_ind or ">" in fm_no_ind:
-        problems.append("frontmatter contains angle brackets (injection risk / not allowed)")
+    # Angle brackets are an injection risk. Checking the PARSED values (rather
+    # than the raw text) means YAML block-scalar indicators like `>-` are never
+    # mistaken for content, so no indicator-stripping hack is needed.
+    for key, value in fm.items():
+        if isinstance(value, str) and ("<" in value or ">" in value):
+            problems.append(f"frontmatter value for {key!r} contains angle brackets "
+                            "(injection risk / not allowed)")
+            break
 
     # Additional top-level keys are PERMITTED by the standard (name+description
     # are the only required fields). They are reported for information, not

tools/validate-skill.py

@@ -10,6 +10,10 @@ import re
 import sys
 import glob
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from skill_frontmatter import load_frontmatter, FrontmatterError
+
 # Kept in sync with the CI workflow (.github/workflows/validate-skills.yml),
 # which now delegates to this script so there is a single source of truth.
 REQUIRED_FIELDS = ["name", "description", "domain", "subdomain", "tags",
@@ -81,104 +85,6 @@ YELLOW = "\033[93m"
 RESET = "\033[0m"
 
 
-def parse_frontmatter(text):
-    """Extract YAML frontmatter as a dict (simple stdlib-only parser).
-
-    Handles the common SKILL.md patterns:
-    - key: scalar value
-    - key: [inline, list]
-    - key:\n  - list\n  - items
-    - key: >-  (folded scalar — content on following indented lines)
-
-    Edge case note: ``list_values`` is reset to ``[]`` whenever a new key
-    with a scalar value is encountered, so a list from a prior block cannot
-    leak into an unrelated key.  The only remaining theoretical edge case is
-    a key with *no* value that is immediately followed by non-list, non-empty
-    lines that look like scalars — those lines are currently ignored (the key
-    is treated as having no value).  This is acceptable for well-formed SKILL.md
-    files and matches the behaviour contributors expect.
-    """
-    if not text.startswith("---"):
-        return None
-    end = text.find("---", 3)
-    if end == -1:
-        return None
-    block = text[3:end].strip()
-    data = {}
-    current_key = None
-    list_values: list = []
-    in_folded = False  # True when we are collecting a YAML >- / > folded scalar
-    folded_lines: list = []
-
-    for line in block.split("\n"):
-        stripped = line.strip()
-
-        # Flush a completed folded scalar when we hit the next top-level key.
-        if in_folded and stripped and not line.startswith(" ") and not line.startswith("\t"):
-            if current_key and folded_lines:
-                data[current_key] = " ".join(folded_lines)
-            in_folded = False
-            folded_lines = []
-            current_key = None
-
-        if in_folded:
-            if stripped:
-                folded_lines.append(stripped)
-            continue
-
-        if not stripped or stripped.startswith("#"):
-            continue
-
-        # Handle list items (must come before key: value to avoid misparse).
-        if stripped.startswith("- ") and current_key:
-            list_values.append(stripped[2:].strip().strip('"').strip("'"))
-            data[current_key] = list(list_values)  # copy so future mutations don't leak
-            continue
-
-        # Only TOP-LEVEL keys (column 0) define frontmatter fields. An indented
-        # ``key: value`` line belongs to a nested structure (e.g. a framework
-        # mapping object that has its own ``name:``/``id:``) and must NOT be
-        # treated as a top-level field — otherwise a nested ``name:`` clobbers
-        # the skill's real ``name``.
-        if line[:1].isspace():
-            continue
-
-        # Handle inline list: tags: [a, b, c]
-        m = re.match(r"^(\w[\w_-]*):\s*\[(.+)\]\s*$", stripped)
-        if m:
-            current_key = m.group(1)
-            items = [i.strip().strip('"').strip("'") for i in m.group(2).split(",")]
-            data[current_key] = items
-            list_values = list(items)
-            continue
-
-        # Handle key: >- or key: > (folded scalar start)
-        m = re.match(r"^(\w[\w_-]*):\s*>[-|]?\s*$", stripped)
-        if m:
-            current_key = m.group(1)
-            list_values = []
-            in_folded = True
-            folded_lines = []
-            continue
-
-        # Handle key: value (plain scalar)
-        m = re.match(r'^(\w[\w_-]*):\s*(.*)$', stripped)
-        if m:
-            current_key = m.group(1)
-            val = m.group(2).strip().strip('"').strip("'")
-            list_values = []  # reset; new scalar key cannot inherit a prior list
-            if val:
-                data[current_key] = val
-            # If val is empty the key is present but value-less (e.g. start of block list)
-            continue
-
-    # Flush any trailing folded scalar.
-    if in_folded and current_key and folded_lines:
-        data[current_key] = " ".join(folded_lines)
-
-    return data
-
-
 def validate_skill(skill_dir):
     """Validate a single skill directory. Returns list of error strings."""
     errors = []
@@ -188,16 +94,11 @@ def validate_skill(skill_dir):
         return [f"SKILL.md not found in {skill_dir}"]
 
     try:
-        with open(skill_md, encoding="utf-8") as f:
-            content = f.read()
+        fm = load_frontmatter(skill_md)
+    except FrontmatterError as e:
+        return [str(e)]
     except IOError as e:
         return [f"Could not read SKILL.md: {e}"]
-    except UnicodeDecodeError as e:
-        return [f"Encoding error in SKILL.md (not valid UTF-8): {e}"]
-
-    fm = parse_frontmatter(content)
-    if fm is None:
-        return ["No valid YAML frontmatter found (must start with ---)"]
 
     # Check required fields.
     for field in REQUIRED_FIELDS: