#!/usr/bin/env python3 """Fluentd/Fluent Bit log forwarding configuration generator and tester.""" import json import argparse import socket import time from datetime import datetime try: from fluent import sender, event HAS_FLUENT = True except ImportError: HAS_FLUENT = False def generate_fluentbit_config(inputs, output_host="127.0.0.1", output_port=24224): """Generate Fluent Bit configuration for log collection and forwarding.""" sections = ["[SERVICE]\n Flush 5\n Daemon Off\n Log_Level info\n Parsers_File parsers.conf\n"] input_configs = { "syslog": "[INPUT]\n Name syslog\n Tag syslog.*\n Listen 0.0.0.0\n Port 5140\n Mode udp\n Parser syslog-rfc3164\n", "tail": "[INPUT]\n Name tail\n Tag file.*\n Path /var/log/*.log\n DB /var/log/flb_tail.db\n Read_from_Head True\n Refresh_Interval 10\n", "systemd": "[INPUT]\n Name systemd\n Tag systemd.*\n Systemd_Filter _SYSTEMD_UNIT=sshd.service\n Read_From_Tail On\n", "tcp": "[INPUT]\n Name tcp\n Tag tcp.*\n Listen 0.0.0.0\n Port 5170\n Format json\n", } for inp in inputs: if inp in input_configs: sections.append(input_configs[inp]) sections.append( "[FILTER]\n" " Name record_modifier\n" " Match *\n" " Record hostname ${HOSTNAME}\n" " Record environment production\n" ) sections.append( "[FILTER]\n" " Name grep\n" " Match syslog.*\n" " Exclude message ^healthcheck\n" ) sections.append( f"[OUTPUT]\n" f" Name forward\n" f" Match *\n" f" Host {output_host}\n" f" Port {output_port}\n" f" Retry_Limit 5\n" ) return "\n".join(sections) def generate_fluentd_config(outputs, bind_port=24224): """Generate Fluentd aggregator configuration.""" sections = [ f"\n" f" @type forward\n" f" port {bind_port}\n" f" bind 0.0.0.0\n" f" \n" f" shared_key fluentd_secure_key_change_me\n" f" self_hostname aggregator.local\n" f" \n" f"\n", ] sections.append( "\n" " @type record_transformer\n" " \n" " received_at ${time}\n" " aggregator_host \"#{Socket.gethostname}\"\n" " \n" "\n" ) output_configs = { "elasticsearch": ( '\n' ' @type elasticsearch\n' ' host elasticsearch.local\n' ' port 9200\n' ' logstash_format true\n' ' logstash_prefix fluentd\n' ' include_tag_key true\n' ' \n' ' @type file\n' ' path /var/log/fluentd/buffer/es\n' ' flush_interval 10s\n' ' chunk_limit_size 8MB\n' ' retry_max_interval 30\n' ' retry_forever true\n' ' \n' '\n' ), "s3": ( '\n' ' @type s3\n' ' s3_bucket security-logs-bucket\n' ' s3_region us-east-1\n' ' path logs/\n' ' time_slice_format %Y%m%d%H\n' ' \n' ' @type file\n' ' path /var/log/fluentd/buffer/s3\n' ' timekey 3600\n' ' timekey_wait 10m\n' ' \n' '\n' ), "splunk": ( '\n' ' @type splunk_hec\n' ' hec_host splunk.local\n' ' hec_port 8088\n' ' hec_token YOUR_HEC_TOKEN\n' ' index main\n' ' source fluentd\n' ' \n' ' @type memory\n' ' flush_interval 5s\n' ' \n' '\n' ), } for out in outputs: if out in output_configs: sections.append(output_configs[out]) return "\n".join(sections) def validate_config(config_text, config_type="fluentbit"): """Validate configuration syntax for common issues.""" errors = [] lines = config_text.split("\n") open_sections = 0 for i, line in enumerate(lines, 1): stripped = line.strip() if stripped.startswith("[") and stripped.endswith("]"): if config_type == "fluentbit": valid_sections = {"[SERVICE]", "[INPUT]", "[FILTER]", "[OUTPUT]", "[PARSER]"} if stripped not in valid_sections: errors.append(f"Line {i}: Unknown section '{stripped}'") if stripped.startswith("<") and not stripped.startswith("