Add agentskills.io compliance schema and validator

- tools/agentskills-skill.schema.json: strict JSON Schema for the
  agentskills.io SKILL.md frontmatter standard (name+description required;
  optional license/compatibility/metadata/allowed-tools; no other top-level
  keys).
- tools/validate-agentskills.py: read-only compliance validator (also checks
  name==directory and the no-angle-brackets rule).
This commit is contained in:
Mahipal
2026-08-02 09:32:13 -07:00
parent 2fb6a9faff
commit 9bd6051d84
2 changed files with 183 additions and 0 deletions
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env python3
"""Validate SKILL.md frontmatter against the strict agentskills.io standard.
Reports, per skill, any deviation from tools/agentskills-skill.schema.json plus
the two constraints JSON Schema can't express (name == parent dir; no angle
brackets in frontmatter). READ-ONLY; never edits files.
Usage:
python3 tools/validate-agentskills.py # summary + report
python3 tools/validate-agentskills.py --json # machine-readable JSON
python3 tools/validate-agentskills.py --strict # exit 1 if any non-compliant
"""
import os, re, sys, json, glob
from collections import Counter
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
ALLOWED = {"name", "description", "license", "compatibility", "metadata", "allowed-tools"}
NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$")
def top_level_keys_and_scalars(fm):
"""Minimal YAML: top-level keys (col 0) + scalar values for name/description."""
keys = []
scalars = {}
lines = fm.split("\n")
for i, line in enumerate(lines):
m = re.match(r"^([A-Za-z0-9_-]+):(.*)$", line)
if not m:
continue
key, rest = m.group(1), m.group(2)
keys.append(key)
val = rest.strip()
if val and val[0] in "|>": # block scalar -> gather following indented lines
buf = []
for nxt in lines[i + 1:]:
if re.match(r"^\s+\S", nxt):
buf.append(nxt.strip())
elif nxt.strip() == "":
buf.append("")
else:
break
val = " ".join(x for x in buf if x != "").strip()
elif not val:
# could be a folded plain scalar wrapped onto continuation lines
buf = []
for nxt in lines[i + 1:]:
if re.match(r"^\s+-\s", nxt) or re.match(r"^[A-Za-z0-9_-]+:", nxt):
break
if re.match(r"^\s+\S", nxt):
buf.append(nxt.strip())
else:
break
val = " ".join(buf).strip()
scalars[key] = val.strip().strip("\"'")
return keys, scalars
def validate(path):
slug = os.path.basename(os.path.dirname(path))
text = open(path, encoding="utf-8").read()
m = re.match(r"^---\n(.*?)\n---", text, re.DOTALL)
problems = []
if not m:
return slug, ["no YAML frontmatter block"], []
fm = m.group(1)
keys, scalars = top_level_keys_and_scalars(fm)
if "name" not in keys:
problems.append("missing required key: name")
if "description" not in keys:
problems.append("missing required key: description")
name = scalars.get("name", "")
if name:
if not NAME_RE.match(name):
problems.append(f"name not lowercase-kebab-case: {name!r}")
if not (1 <= len(name) <= 64):
problems.append(f"name length {len(name)} out of 1..64")
if name != slug:
problems.append(f"name {name!r} != directory {slug!r}")
desc = scalars.get("description", "")
if desc:
if not (1 <= len(desc) <= 1024):
problems.append(f"description length {len(desc)} out of 1..1024")
elif "description" in keys:
problems.append("description empty")
if "<" in fm or ">" in fm:
problems.append("frontmatter contains angle brackets (injection risk / not allowed)")
nonstd = [k for k in keys if k not in ALLOWED]
for k in nonstd:
problems.append(f"non-standard top-level key: {k}")
return slug, problems, nonstd
def main():
as_json = "--json" in sys.argv
strict = "--strict" in sys.argv
skills = sorted(glob.glob(os.path.join(REPO, "skills", "*", "SKILL.md")))
results = []
nonstd_hist = Counter()
compliant = 0
for p in skills:
slug, problems, nonstd = validate(p)
nonstd_hist.update(nonstd)
if not problems:
compliant += 1
results.append({"skill": slug, "compliant": not problems, "problems": problems})
noncompliant = [r for r in results if not r["compliant"]]
summary = {
"total": len(skills),
"compliant": compliant,
"noncompliant": len(noncompliant),
"nonstandard_key_frequency": dict(nonstd_hist.most_common()),
}
if as_json:
print(json.dumps({"summary": summary, "results": results}, indent=1))
else:
print(f"agentskills.io compliance: {compliant}/{len(skills)} compliant, "
f"{len(noncompliant)} non-compliant")
print("\nNon-standard top-level keys (count of skills carrying each):")
for k, n in nonstd_hist.most_common():
print(f" {k:20s} {n}")
# distinct problem types (excluding the per-key nonstd noise)
other = Counter()
for r in noncompliant:
for pr in r["problems"]:
if not pr.startswith("non-standard top-level key:"):
other[re.sub(r':.*$', '', pr)] += 1
if other:
print("\nOther (non-key) issues:")
for k, n in other.most_common():
print(f" {k}: {n}")
if strict and noncompliant:
sys.exit(1)
if __name__ == "__main__":
main()