Anthropic-Cybersecurity-Skills/skills/defending-llms-with-guardrails/scripts/agent.py

#!/usr/bin/env python3
"""Guardrail validation harness.

Runs a corpus of labeled prompts through LLM Guard or Llama Guard 3 and reports
block rate, false-positive rate, and per-scanner verdicts. The corpus is a JSONL
file where each line is: {"prompt": "...", "label": "unsafe"|"safe"}.

Examples
--------
    python agent.py llmguard   --input payloads.jsonl --report report.json
    python agent.py llamaguard --model meta-llama/Llama-Guard-3-8B --input payloads.jsonl
"""
import argparse
import json
import sys
from pathlib import Path


def load_corpus(path: str):
    rows = []
    with open(path, "r", encoding="utf-8") as fh:
        for i, line in enumerate(fh, 1):
            line = line.strip()
            if not line:
                continue
            try:
                obj = json.loads(line)
            except json.JSONDecodeError as exc:
                print(f"[!] line {i}: invalid JSON ({exc})", file=sys.stderr)
                continue
            if "prompt" not in obj:
                print(f"[!] line {i}: missing 'prompt'", file=sys.stderr)
                continue
            rows.append({"prompt": obj["prompt"], "label": obj.get("label", "unknown")})
    return rows


def run_llmguard(corpus):
    try:
        from llm_guard import scan_prompt
        from llm_guard.input_scanners import PromptInjection, Toxicity, Secrets, TokenLimit
        from llm_guard.input_scanners.prompt_injection import MatchType
    except ImportError:
        sys.exit("[!] llm-guard not installed. Run: pip install llm-guard")

    scanners = [
        PromptInjection(threshold=0.5, match_type=MatchType.FULL),
        Toxicity(threshold=0.5),
        Secrets(),
        TokenLimit(limit=4096),
    ]
    results = []
    for row in corpus:
        try:
            _, valid, score = scan_prompt(scanners, row["prompt"])
            blocked = any(not v for v in valid.values())
        except Exception as exc:  # scanner runtime error
            print(f"[!] scan error: {exc}", file=sys.stderr)
            blocked, valid, score = False, {}, {}
        results.append({
            "label": row["label"],
            "blocked": blocked,
            "verdicts": valid,
            "scores": score,
        })
    return results


def run_llamaguard(corpus, model_id):
    try:
        import torch
        from transformers import AutoTokenizer, AutoModelForCausalLM
    except ImportError:
        sys.exit("[!] transformers/torch not installed. Run: pip install 'transformers>=4.43' torch accelerate")

    print(f"[*] loading {model_id} ...", file=sys.stderr)
    tok = AutoTokenizer.from_pretrained(model_id)
    model = AutoModelForCausalLM.from_pretrained(
        model_id, torch_dtype=torch.bfloat16, device_map="auto"
    )

    def moderate(prompt):
        chat = [{"role": "user", "content": prompt}]
        ids = tok.apply_chat_template(chat, return_tensors="pt").to(model.device)
        out = model.generate(input_ids=ids, max_new_tokens=100, pad_token_id=0)
        return tok.decode(out[0][ids.shape[-1]:], skip_special_tokens=True).strip()

    results = []
    for row in corpus:
        verdict = moderate(row["prompt"])
        blocked = verdict.lower().startswith("unsafe")
        category = verdict.split("\n", 1)[1] if "\n" in verdict else ""
        results.append({
            "label": row["label"],
            "blocked": blocked,
            "verdict": verdict,
            "category": category,
        })
    return results


def summarize(results):
    total = len(results)
    if total == 0:
        return {"total": 0}
    unsafe = [r for r in results if r["label"] == "unsafe"]
    safe = [r for r in results if r["label"] == "safe"]
    tp = sum(1 for r in unsafe if r["blocked"])
    fn = sum(1 for r in unsafe if not r["blocked"])
    fp = sum(1 for r in safe if r["blocked"])
    tn = sum(1 for r in safe if not r["blocked"])
    return {
        "total": total,
        "unsafe_total": len(unsafe),
        "safe_total": len(safe),
        "true_positive_blocked": tp,
        "false_negative_missed": fn,
        "false_positive_overblock": fp,
        "true_negative_allowed": tn,
        "block_rate": round(tp / len(unsafe), 3) if unsafe else None,
        "false_positive_rate": round(fp / len(safe), 3) if safe else None,
    }


def main():
    p = argparse.ArgumentParser(description="Guardrail validation harness")
    p.add_argument("engine", choices=["llmguard", "llamaguard"], help="guardrail engine to test")
    p.add_argument("--input", required=True, help="JSONL corpus of {prompt,label}")
    p.add_argument("--model", default="meta-llama/Llama-Guard-3-8B", help="Llama Guard model id")
    p.add_argument("--report", help="write detailed JSON report to this path")
    args = p.parse_args()

    if not Path(args.input).is_file():
        sys.exit(f"[!] input not found: {args.input}")

    corpus = load_corpus(args.input)
    if not corpus:
        sys.exit("[!] corpus is empty")
    print(f"[*] loaded {len(corpus)} prompts", file=sys.stderr)

    if args.engine == "llmguard":
        results = run_llmguard(corpus)
    else:
        results = run_llamaguard(corpus, args.model)

    summary = summarize(results)
    print(json.dumps(summary, indent=2))

    if args.report:
        with open(args.report, "w", encoding="utf-8") as fh:
            json.dump({"summary": summary, "results": results}, fh, indent=2)
        print(f"[+] detailed report written to {args.report}", file=sys.stderr)


if __name__ == "__main__":
    main()