Anthropic-Cybersecurity-Skills/skills/building-threat-feed-aggregation-with-misp/scripts/agent.py

#!/usr/bin/env python3
"""Threat Feed Aggregation Agent - Aggregates and correlates threat intelligence feeds using MISP."""

import json
import logging
import os
import argparse
from datetime import datetime
from collections import defaultdict

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger(__name__)


def misp_request(url, key, endpoint, method="GET", data=None):
    """Make authenticated MISP API request."""
    headers = {"Authorization": key, "Accept": "application/json", "Content-Type": "application/json"}
    full_url = f"{url}/{endpoint}"
    try:
        if method == "GET":
            resp = requests.get(full_url, headers=headers, timeout=30, verify=not os.environ.get("SKIP_TLS_VERIFY", "").lower() == "true")  # Set SKIP_TLS_VERIFY=true for self-signed certs in lab environments
        else:
            resp = requests.post(full_url, headers=headers, json=data or {}, timeout=30, verify=not os.environ.get("SKIP_TLS_VERIFY", "").lower() == "true")  # Set SKIP_TLS_VERIFY=true for self-signed certs in lab environments
        resp.raise_for_status()
        return resp.json()
    except requests.RequestException as e:
        logger.error("MISP request failed: %s", e)
        return {"error": str(e)}


def list_feeds(url, key):
    """List configured MISP feeds."""
    data = misp_request(url, key, "feeds/index", method="POST")
    feeds = data if isinstance(data, list) else data.get("Feed", [])
    result = []
    for feed in feeds:
        f = feed.get("Feed", feed) if isinstance(feed, dict) and "Feed" in feed else feed
        result.append({"id": f.get("id"), "name": f.get("name"), "provider": f.get("provider"),
                       "url": f.get("url"), "enabled": f.get("enabled"), "source_format": f.get("source_format"),
                       "caching_enabled": f.get("caching_enabled")})
    logger.info("Found %d configured feeds", len(result))
    return result


def fetch_feed_data(url, key, feed_id):
    """Fetch and cache data from a specific feed."""
    result = misp_request(url, key, f"feeds/cacheFeeds/{feed_id}", method="POST")
    logger.info("Cached feed %s", feed_id)
    return result


def search_attributes(url, key, attr_type=None, value=None, last_days=30):
    """Search MISP attributes across all events."""
    search_body = {"returnFormat": "json", "limit": 1000, "last": f"{last_days}d"}
    if attr_type:
        search_body["type"] = attr_type
    if value:
        search_body["value"] = value
    data = misp_request(url, key, "attributes/restSearch", method="POST", data=search_body)
    attributes = data.get("response", {}).get("Attribute", [])
    logger.info("Found %d attributes (type=%s, last %dd)", len(attributes), attr_type, last_days)
    return attributes


def aggregate_feed_statistics(url, key, last_days=30):
    """Aggregate statistics across all feeds."""
    events_data = misp_request(url, key, "events/restSearch", method="POST",
                                data={"returnFormat": "json", "limit": 500, "last": f"{last_days}d"})
    events = events_data.get("response", [])
    stats = {"total_events": len(events), "by_threat_level": defaultdict(int),
             "by_org": defaultdict(int), "by_tag": defaultdict(int), "attribute_types": defaultdict(int)}
    threat_levels = {"1": "High", "2": "Medium", "3": "Low", "4": "Undefined"}
    for event_wrap in events:
        event = event_wrap.get("Event", event_wrap)
        tl = threat_levels.get(str(event.get("threat_level_id", 4)), "Undefined")
        stats["by_threat_level"][tl] += 1
        org = event.get("Orgc", {}).get("name", "Unknown")
        stats["by_org"][org] += 1
        for tag in event.get("Tag", []):
            stats["by_tag"][tag.get("name", "")] += 1
        for attr in event.get("Attribute", []):
            stats["attribute_types"][attr.get("type", "unknown")] += 1
    return {k: dict(v) if isinstance(v, defaultdict) else v for k, v in stats.items()}


def correlate_across_feeds(url, key, ioc_value):
    """Correlate an IOC across all feed events."""
    data = misp_request(url, key, "attributes/restSearch", method="POST",
                        data={"returnFormat": "json", "value": ioc_value, "limit": 100})
    attributes = data.get("response", {}).get("Attribute", [])
    correlations = []
    seen_events = set()
    for attr in attributes:
        event_id = attr.get("event_id")
        if event_id not in seen_events:
            seen_events.add(event_id)
            correlations.append({"event_id": event_id, "type": attr.get("type"), "category": attr.get("category"),
                                 "comment": attr.get("comment", "")[:100]})
    logger.info("IOC '%s' found in %d events", ioc_value, len(correlations))
    return correlations


def assess_feed_health(feeds):
    """Assess health and coverage of configured feeds."""
    total = len(feeds)
    enabled = sum(1 for f in feeds if f.get("enabled"))
    cached = sum(1 for f in feeds if f.get("caching_enabled"))
    return {"total_feeds": total, "enabled": enabled, "disabled": total - enabled,
            "caching_enabled": cached, "health_score": round(enabled / total * 100, 1) if total else 0}


def generate_report(feeds, stats, feed_health):
    """Generate threat feed aggregation report."""
    report = {
        "timestamp": datetime.utcnow().isoformat(),
        "feed_inventory": feeds,
        "feed_health": feed_health,
        "aggregated_statistics": stats,
    }
    print(f"FEED REPORT: {feed_health['total_feeds']} feeds, {feed_health['enabled']} enabled, "
          f"{stats.get('total_events', 0)} events")
    return report


def main():
    parser = argparse.ArgumentParser(description="Threat Feed Aggregation with MISP")
    parser.add_argument("--url", required=True, help="MISP instance URL")
    parser.add_argument("--key", required=True, help="MISP API key")
    parser.add_argument("--days", type=int, default=30, help="Look-back period in days")
    parser.add_argument("--correlate", help="IOC value to correlate across feeds")
    parser.add_argument("--output", default="feed_aggregation_report.json")
    args = parser.parse_args()

    feeds = list_feeds(args.url, args.key)
    stats = aggregate_feed_statistics(args.url, args.key, args.days)
    feed_health = assess_feed_health(feeds)
    report = generate_report(feeds, stats, feed_health)
    if args.correlate:
        report["correlation_results"] = correlate_across_feeds(args.url, args.key, args.correlate)
    with open(args.output, "w") as f:
        json.dump(report, f, indent=2)
    logger.info("Report saved to %s", args.output)


if __name__ == "__main__":
    main()