mirror of
https://github.com/mukul975/Anthropic-Cybersecurity-Skills.git
synced 2026-08-08 11:23:20 +03:00
Add 30 new production-grade cybersecurity skills: AI security, supply chain, firmware, cloud-native, compliance, deception, crypto, threat hunting, purple team, OT, privacy
This commit is contained in:
mukul975
@@ -0,0 +1,807 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Agent for performing cloud log forensics using AWS Athena.
|
||||
|
||||
Automates Athena table creation with partition projection and runs forensic
|
||||
SQL queries against CloudTrail, VPC Flow Logs, S3 access logs, and ALB logs.
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import argparse
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
CLOUDTRAIL_DDL = """
|
||||
CREATE EXTERNAL TABLE IF NOT EXISTS {database}.cloudtrail_logs (
|
||||
eventVersion STRING,
|
||||
userIdentity STRUCT<
|
||||
type: STRING,
|
||||
principalId: STRING,
|
||||
arn: STRING,
|
||||
accountId: STRING,
|
||||
invokedBy: STRING,
|
||||
accessKeyId: STRING,
|
||||
userName: STRING,
|
||||
sessionContext: STRUCT<
|
||||
attributes: STRUCT<
|
||||
mfaAuthenticated: STRING,
|
||||
creationDate: STRING>,
|
||||
sessionIssuer: STRUCT<
|
||||
type: STRING,
|
||||
principalId: STRING,
|
||||
arn: STRING,
|
||||
accountId: STRING,
|
||||
userName: STRING>,
|
||||
ec2RoleDelivery: STRING,
|
||||
webIdFederationData: STRUCT<
|
||||
federatedProvider: STRING,
|
||||
attributes: MAP<STRING, STRING>>>>,
|
||||
eventTime STRING,
|
||||
eventSource STRING,
|
||||
eventName STRING,
|
||||
awsRegion STRING,
|
||||
sourceIPAddress STRING,
|
||||
userAgent STRING,
|
||||
errorCode STRING,
|
||||
errorMessage STRING,
|
||||
requestParameters STRING,
|
||||
responseElements STRING,
|
||||
additionalEventData STRING,
|
||||
requestId STRING,
|
||||
eventId STRING,
|
||||
readOnly STRING,
|
||||
resources ARRAY<STRUCT<
|
||||
arn: STRING,
|
||||
accountId: STRING,
|
||||
type: STRING>>,
|
||||
eventType STRING,
|
||||
apiVersion STRING,
|
||||
recipientAccountId STRING,
|
||||
serviceEventDetails STRING,
|
||||
sharedEventID STRING,
|
||||
vpcEndpointId STRING,
|
||||
tlsDetails STRUCT<
|
||||
tlsVersion: STRING,
|
||||
cipherSuite: STRING,
|
||||
clientProvidedHostHeader: STRING>
|
||||
)
|
||||
COMMENT 'CloudTrail logs with partition projection for forensic analysis'
|
||||
PARTITIONED BY (
|
||||
`account` STRING,
|
||||
`region` STRING,
|
||||
`timestamp` STRING
|
||||
)
|
||||
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
|
||||
STORED AS INPUTFORMAT 'com.amazon.emr.cloudtrail.CloudTrailInputFormat'
|
||||
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
|
||||
LOCATION 's3://{bucket}/AWSLogs/'
|
||||
TBLPROPERTIES (
|
||||
'projection.enabled' = 'true',
|
||||
'projection.account.type' = 'enum',
|
||||
'projection.account.values' = '{account_id}',
|
||||
'projection.region.type' = 'enum',
|
||||
'projection.region.values' = '{regions}',
|
||||
'projection.timestamp.type' = 'date',
|
||||
'projection.timestamp.format' = 'yyyy/MM/dd',
|
||||
'projection.timestamp.range' = '2020/01/01,NOW',
|
||||
'projection.timestamp.interval' = '1',
|
||||
'projection.timestamp.interval.unit' = 'DAYS',
|
||||
'storage.location.template' = 's3://{bucket}/AWSLogs/${{account}}/CloudTrail/${{region}}/${{timestamp}}'
|
||||
)
|
||||
"""
|
||||
|
||||
VPC_FLOW_DDL = """
|
||||
CREATE EXTERNAL TABLE IF NOT EXISTS {database}.vpc_flow_logs (
|
||||
version INT,
|
||||
account_id STRING,
|
||||
interface_id STRING,
|
||||
srcaddr STRING,
|
||||
dstaddr STRING,
|
||||
srcport INT,
|
||||
dstport INT,
|
||||
protocol BIGINT,
|
||||
packets BIGINT,
|
||||
bytes BIGINT,
|
||||
start BIGINT,
|
||||
`end` BIGINT,
|
||||
action STRING,
|
||||
log_status STRING,
|
||||
vpc_id STRING,
|
||||
subnet_id STRING,
|
||||
az_id STRING,
|
||||
sublocation_type STRING,
|
||||
sublocation_id STRING,
|
||||
pkt_srcaddr STRING,
|
||||
pkt_dstaddr STRING,
|
||||
region STRING,
|
||||
pkt_src_aws_service STRING,
|
||||
pkt_dst_aws_service STRING,
|
||||
flow_direction STRING,
|
||||
traffic_path INT
|
||||
)
|
||||
PARTITIONED BY (
|
||||
`date` STRING
|
||||
)
|
||||
ROW FORMAT DELIMITED
|
||||
FIELDS TERMINATED BY ' '
|
||||
LOCATION 's3://{bucket}/AWSLogs/{account_id}/vpcflowlogs/'
|
||||
TBLPROPERTIES (
|
||||
'skip.header.line.count' = '1',
|
||||
'projection.enabled' = 'true',
|
||||
'projection.date.type' = 'date',
|
||||
'projection.date.format' = 'yyyy/MM/dd',
|
||||
'projection.date.range' = '2020/01/01,NOW',
|
||||
'projection.date.interval' = '1',
|
||||
'projection.date.interval.unit' = 'DAYS',
|
||||
'storage.location.template' = 's3://{bucket}/AWSLogs/{account_id}/vpcflowlogs/{primary_region}/${{date}}'
|
||||
)
|
||||
"""
|
||||
|
||||
S3_ACCESS_DDL = """
|
||||
CREATE EXTERNAL TABLE IF NOT EXISTS {database}.s3_access_logs (
|
||||
bucket_owner STRING,
|
||||
bucket_name STRING,
|
||||
request_datetime STRING,
|
||||
remote_ip STRING,
|
||||
requester STRING,
|
||||
request_id STRING,
|
||||
operation STRING,
|
||||
key STRING,
|
||||
request_uri STRING,
|
||||
http_status INT,
|
||||
error_code STRING,
|
||||
bytes_sent BIGINT,
|
||||
object_size BIGINT,
|
||||
total_time INT,
|
||||
turn_around_time INT,
|
||||
referrer STRING,
|
||||
user_agent STRING,
|
||||
version_id STRING,
|
||||
host_id STRING,
|
||||
signature_version STRING,
|
||||
cipher_suite STRING,
|
||||
authentication_type STRING,
|
||||
host_header STRING,
|
||||
tls_version STRING,
|
||||
access_point_arn STRING,
|
||||
acl_required STRING
|
||||
)
|
||||
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe'
|
||||
WITH SERDEPROPERTIES (
|
||||
'serialization.format' = '1',
|
||||
'input.regex' = '([^ ]*) ([^ ]*) \\\\[(.*?)\\\\] ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) (\"[^\"]*\"|-) (-|[0-9]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) (\"[^\"]*\"|-) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*)'
|
||||
)
|
||||
STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
|
||||
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
|
||||
LOCATION 's3://{bucket}/logs/'
|
||||
"""
|
||||
|
||||
ALB_ACCESS_DDL = """
|
||||
CREATE EXTERNAL TABLE IF NOT EXISTS {database}.alb_access_logs (
|
||||
type STRING,
|
||||
time STRING,
|
||||
elb STRING,
|
||||
client_ip STRING,
|
||||
client_port INT,
|
||||
target_ip STRING,
|
||||
target_port INT,
|
||||
request_processing_time DOUBLE,
|
||||
target_processing_time DOUBLE,
|
||||
response_processing_time DOUBLE,
|
||||
elb_status_code INT,
|
||||
target_status_code STRING,
|
||||
received_bytes BIGINT,
|
||||
sent_bytes BIGINT,
|
||||
request_verb STRING,
|
||||
request_url STRING,
|
||||
request_proto STRING,
|
||||
user_agent STRING,
|
||||
ssl_cipher STRING,
|
||||
ssl_protocol STRING,
|
||||
target_group_arn STRING,
|
||||
trace_id STRING,
|
||||
domain_name STRING,
|
||||
chosen_cert_arn STRING,
|
||||
matched_rule_priority STRING,
|
||||
request_creation_time STRING,
|
||||
actions_executed STRING,
|
||||
redirect_url STRING,
|
||||
lambda_error_reason STRING,
|
||||
target_port_list STRING,
|
||||
target_status_code_list STRING,
|
||||
classification STRING,
|
||||
classification_reason STRING,
|
||||
conn_trace_id STRING
|
||||
)
|
||||
PARTITIONED BY (
|
||||
`day` STRING
|
||||
)
|
||||
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe'
|
||||
WITH SERDEPROPERTIES (
|
||||
'serialization.format' = '1',
|
||||
'input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[0-9]*) (-|[0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^ ]*)\" \"([^ ]*)\"'
|
||||
)
|
||||
STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
|
||||
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
|
||||
LOCATION 's3://{bucket}/AWSLogs/{account_id}/elasticloadbalancing/{primary_region}/'
|
||||
TBLPROPERTIES (
|
||||
'projection.enabled' = 'true',
|
||||
'projection.day.type' = 'date',
|
||||
'projection.day.format' = 'yyyy/MM/dd',
|
||||
'projection.day.range' = '2020/01/01,NOW',
|
||||
'projection.day.interval' = '1',
|
||||
'projection.day.interval.unit' = 'DAYS',
|
||||
'storage.location.template' = 's3://{bucket}/AWSLogs/{account_id}/elasticloadbalancing/{primary_region}/${{day}}'
|
||||
)
|
||||
"""
|
||||
|
||||
FORENSIC_QUERIES = {
|
||||
"unauthorized_access": """
|
||||
SELECT
|
||||
eventtime,
|
||||
useridentity.arn AS caller_arn,
|
||||
useridentity.accountid AS account,
|
||||
eventsource,
|
||||
eventname,
|
||||
errorcode,
|
||||
errormessage,
|
||||
sourceipaddress,
|
||||
useragent
|
||||
FROM {database}.cloudtrail_logs
|
||||
WHERE errorcode IN ('AccessDenied', 'UnauthorizedAccess', 'Client.UnauthorizedAccess')
|
||||
AND timestamp BETWEEN '{start_date}' AND '{end_date}'
|
||||
ORDER BY eventtime DESC
|
||||
LIMIT 1000
|
||||
""",
|
||||
"privilege_escalation": """
|
||||
SELECT
|
||||
eventtime,
|
||||
useridentity.arn AS actor,
|
||||
eventname,
|
||||
eventsource,
|
||||
CASE
|
||||
WHEN eventname IN ('AttachUserPolicy','AttachRolePolicy','AttachGroupPolicy')
|
||||
THEN json_extract_scalar(requestparameters, '$.policyArn')
|
||||
WHEN eventname IN ('CreateAccessKey','CreateLoginProfile','UpdateLoginProfile')
|
||||
THEN json_extract_scalar(requestparameters, '$.userName')
|
||||
WHEN eventname = 'AssumeRole'
|
||||
THEN json_extract_scalar(requestparameters, '$.roleArn')
|
||||
ELSE requestparameters
|
||||
END AS target_resource,
|
||||
sourceipaddress,
|
||||
errorcode
|
||||
FROM {database}.cloudtrail_logs
|
||||
WHERE eventname IN (
|
||||
'AttachUserPolicy', 'AttachRolePolicy', 'AttachGroupPolicy',
|
||||
'PutUserPolicy', 'PutRolePolicy', 'PutGroupPolicy',
|
||||
'CreatePolicyVersion', 'SetDefaultPolicyVersion',
|
||||
'AddUserToGroup', 'UpdateAssumeRolePolicy',
|
||||
'CreateAccessKey', 'CreateLoginProfile',
|
||||
'UpdateLoginProfile', 'AssumeRole',
|
||||
'PassRole', 'CreateRole'
|
||||
)
|
||||
AND timestamp BETWEEN '{start_date}' AND '{end_date}'
|
||||
ORDER BY eventtime DESC
|
||||
""",
|
||||
"data_exfiltration_s3": """
|
||||
SELECT
|
||||
eventtime,
|
||||
useridentity.arn AS actor,
|
||||
eventname,
|
||||
json_extract_scalar(requestparameters, '$.bucketName') AS bucket,
|
||||
json_extract_scalar(requestparameters, '$.key') AS object_key,
|
||||
sourceipaddress,
|
||||
useragent
|
||||
FROM {database}.cloudtrail_logs
|
||||
WHERE eventsource = 's3.amazonaws.com'
|
||||
AND eventname IN ('GetObject', 'CopyObject', 'SelectObjectContent',
|
||||
'PutBucketPolicy', 'PutBucketAcl', 'PutObjectAcl',
|
||||
'DeleteBucketEncryption', 'PutBucketPublicAccessBlock')
|
||||
AND sourceipaddress NOT LIKE '10.%'
|
||||
AND sourceipaddress NOT LIKE '172.1%'
|
||||
AND sourceipaddress NOT LIKE '172.2%'
|
||||
AND sourceipaddress NOT LIKE '172.3%'
|
||||
AND sourceipaddress NOT LIKE '192.168.%'
|
||||
AND timestamp BETWEEN '{start_date}' AND '{end_date}'
|
||||
ORDER BY eventtime DESC
|
||||
""",
|
||||
"lateral_movement_vpc": """
|
||||
SELECT
|
||||
srcaddr,
|
||||
dstaddr,
|
||||
dstport,
|
||||
protocol,
|
||||
SUM(packets) AS total_packets,
|
||||
SUM(bytes) AS total_bytes,
|
||||
COUNT(*) AS connection_count,
|
||||
MIN(from_unixtime(start)) AS first_seen,
|
||||
MAX(from_unixtime("end")) AS last_seen
|
||||
FROM {database}.vpc_flow_logs
|
||||
WHERE action = 'ACCEPT'
|
||||
AND srcaddr LIKE '10.%'
|
||||
AND dstport IN (22, 3389, 5985, 5986, 445, 135, 139, 5900, 4444, 8080)
|
||||
AND date BETWEEN '{start_date}' AND '{end_date}'
|
||||
GROUP BY srcaddr, dstaddr, dstport, protocol
|
||||
HAVING COUNT(*) > 50
|
||||
ORDER BY connection_count DESC
|
||||
""",
|
||||
"port_scanning": """
|
||||
SELECT
|
||||
srcaddr,
|
||||
COUNT(DISTINCT dstport) AS unique_ports_scanned,
|
||||
COUNT(DISTINCT dstaddr) AS unique_targets,
|
||||
SUM(packets) AS total_packets,
|
||||
MIN(from_unixtime(start)) AS first_seen,
|
||||
MAX(from_unixtime("end")) AS last_seen
|
||||
FROM {database}.vpc_flow_logs
|
||||
WHERE action = 'REJECT'
|
||||
AND date BETWEEN '{start_date}' AND '{end_date}'
|
||||
GROUP BY srcaddr
|
||||
HAVING COUNT(DISTINCT dstport) > 25
|
||||
ORDER BY unique_ports_scanned DESC
|
||||
""",
|
||||
"s3_bulk_download": """
|
||||
SELECT
|
||||
remote_ip,
|
||||
requester,
|
||||
bucket_name,
|
||||
COUNT(*) AS request_count,
|
||||
SUM(bytes_sent) AS total_bytes_downloaded,
|
||||
COUNT(DISTINCT key) AS unique_objects,
|
||||
MIN(request_datetime) AS first_request,
|
||||
MAX(request_datetime) AS last_request
|
||||
FROM {database}.s3_access_logs
|
||||
WHERE operation LIKE '%GET%'
|
||||
AND http_status = 200
|
||||
GROUP BY remote_ip, requester, bucket_name
|
||||
HAVING COUNT(*) > 500
|
||||
ORDER BY total_bytes_downloaded DESC
|
||||
""",
|
||||
"alb_injection_attempts": """
|
||||
SELECT
|
||||
time,
|
||||
client_ip,
|
||||
request_verb,
|
||||
request_url,
|
||||
elb_status_code,
|
||||
target_status_code,
|
||||
user_agent
|
||||
FROM {database}.alb_access_logs
|
||||
WHERE (
|
||||
request_url LIKE '%UNION%SELECT%'
|
||||
OR request_url LIKE '%<script%'
|
||||
OR request_url LIKE '%../../../%'
|
||||
OR request_url LIKE '%/etc/passwd%'
|
||||
OR request_url LIKE '%cmd.exe%'
|
||||
OR request_url LIKE '%/proc/self%'
|
||||
OR request_url LIKE '%SLEEP(%'
|
||||
OR request_url LIKE '%WAITFOR%'
|
||||
OR request_url LIKE '%0x%'
|
||||
OR request_url LIKE '%/admin%'
|
||||
)
|
||||
AND day BETWEEN '{start_date}' AND '{end_date}'
|
||||
ORDER BY time DESC
|
||||
""",
|
||||
"console_login_anomalies": """
|
||||
SELECT
|
||||
eventtime,
|
||||
useridentity.arn AS user_arn,
|
||||
useridentity.username AS username,
|
||||
sourceipaddress,
|
||||
useragent,
|
||||
json_extract_scalar(responseelements, '$.ConsoleLogin') AS login_result,
|
||||
json_extract_scalar(additionaleventdata, '$.MFAUsed') AS mfa_used,
|
||||
json_extract_scalar(additionaleventdata, '$.LoginTo') AS login_target
|
||||
FROM {database}.cloudtrail_logs
|
||||
WHERE eventname = 'ConsoleLogin'
|
||||
AND timestamp BETWEEN '{start_date}' AND '{end_date}'
|
||||
ORDER BY eventtime DESC
|
||||
""",
|
||||
"security_group_changes": """
|
||||
SELECT
|
||||
eventtime,
|
||||
useridentity.arn AS actor,
|
||||
eventname,
|
||||
json_extract_scalar(requestparameters, '$.groupId') AS security_group_id,
|
||||
requestparameters,
|
||||
sourceipaddress
|
||||
FROM {database}.cloudtrail_logs
|
||||
WHERE eventname IN (
|
||||
'AuthorizeSecurityGroupIngress', 'AuthorizeSecurityGroupEgress',
|
||||
'RevokeSecurityGroupIngress', 'RevokeSecurityGroupEgress',
|
||||
'CreateSecurityGroup', 'DeleteSecurityGroup',
|
||||
'ModifySecurityGroupRules'
|
||||
)
|
||||
AND timestamp BETWEEN '{start_date}' AND '{end_date}'
|
||||
ORDER BY eventtime DESC
|
||||
""",
|
||||
"cross_log_correlation": """
|
||||
WITH suspicious_ips AS (
|
||||
SELECT DISTINCT sourceipaddress AS ip
|
||||
FROM {database}.cloudtrail_logs
|
||||
WHERE errorcode IN ('AccessDenied', 'UnauthorizedAccess')
|
||||
AND timestamp BETWEEN '{start_date}' AND '{end_date}'
|
||||
)
|
||||
SELECT
|
||||
v.srcaddr,
|
||||
v.dstaddr,
|
||||
v.dstport,
|
||||
v.protocol,
|
||||
SUM(v.bytes) AS total_bytes,
|
||||
COUNT(*) AS flow_count,
|
||||
MIN(from_unixtime(v.start)) AS first_seen,
|
||||
MAX(from_unixtime(v."end")) AS last_seen
|
||||
FROM {database}.vpc_flow_logs v
|
||||
JOIN suspicious_ips s ON v.srcaddr = s.ip
|
||||
WHERE v.date BETWEEN '{start_date}' AND '{end_date}'
|
||||
GROUP BY v.srcaddr, v.dstaddr, v.dstport, v.protocol
|
||||
ORDER BY total_bytes DESC
|
||||
LIMIT 500
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
class AthenaForensicsAgent:
|
||||
"""Runs forensic investigations against AWS logs via Athena."""
|
||||
|
||||
def __init__(self, database, output_location, region="us-east-1"):
|
||||
self.database = database
|
||||
self.output_location = output_location
|
||||
self.region = region
|
||||
self.client = boto3.client("athena", region_name=region)
|
||||
self.s3_client = boto3.client("s3", region_name=region)
|
||||
|
||||
def execute_query(self, query, wait=True, timeout=300):
|
||||
"""Execute an Athena query and optionally wait for results."""
|
||||
response = self.client.start_query_execution(
|
||||
QueryString=query,
|
||||
QueryExecutionContext={"Database": self.database},
|
||||
ResultConfiguration={"OutputLocation": self.output_location},
|
||||
)
|
||||
execution_id = response["QueryExecutionId"]
|
||||
print(f"[+] Query execution started: {execution_id}")
|
||||
|
||||
if not wait:
|
||||
return execution_id
|
||||
|
||||
start_time = time.time()
|
||||
while True:
|
||||
status = self.client.get_query_execution(QueryExecutionId=execution_id)
|
||||
state = status["QueryExecution"]["Status"]["State"]
|
||||
|
||||
if state == "SUCCEEDED":
|
||||
print(f"[+] Query completed in {time.time() - start_time:.1f}s")
|
||||
return self._get_results(execution_id)
|
||||
elif state in ("FAILED", "CANCELLED"):
|
||||
reason = status["QueryExecution"]["Status"].get(
|
||||
"StateChangeReason", "Unknown"
|
||||
)
|
||||
print(f"[-] Query {state}: {reason}")
|
||||
return None
|
||||
|
||||
if time.time() - start_time > timeout:
|
||||
print(f"[-] Query timed out after {timeout}s")
|
||||
self.client.stop_query_execution(QueryExecutionId=execution_id)
|
||||
return None
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
def _get_results(self, execution_id):
|
||||
"""Fetch query results and return as list of dicts."""
|
||||
results = []
|
||||
paginator = self.client.get_paginator("get_query_results")
|
||||
page_iterator = paginator.paginate(QueryExecutionId=execution_id)
|
||||
|
||||
headers = None
|
||||
for page in page_iterator:
|
||||
rows = page["ResultSet"]["Rows"]
|
||||
for i, row in enumerate(rows):
|
||||
values = [col.get("VarCharValue", "") for col in row["Data"]]
|
||||
if headers is None:
|
||||
headers = values
|
||||
continue
|
||||
results.append(dict(zip(headers, values)))
|
||||
|
||||
print(f"[+] Retrieved {len(results)} rows")
|
||||
return results
|
||||
|
||||
def setup_database(self):
|
||||
"""Create the forensics database if it does not exist."""
|
||||
query = f"CREATE DATABASE IF NOT EXISTS {self.database}"
|
||||
self.execute_query(query)
|
||||
print(f"[+] Database '{self.database}' ready")
|
||||
|
||||
def create_cloudtrail_table(self, bucket, account_id, regions):
|
||||
"""Create CloudTrail table with partition projection."""
|
||||
ddl = CLOUDTRAIL_DDL.format(
|
||||
database=self.database,
|
||||
bucket=bucket,
|
||||
account_id=account_id,
|
||||
regions=",".join(regions) if isinstance(regions, list) else regions,
|
||||
)
|
||||
self.execute_query(ddl)
|
||||
print("[+] CloudTrail table created with partition projection")
|
||||
|
||||
def create_vpc_flow_table(self, bucket, account_id, primary_region):
|
||||
"""Create VPC Flow Logs table with partition projection."""
|
||||
ddl = VPC_FLOW_DDL.format(
|
||||
database=self.database,
|
||||
bucket=bucket,
|
||||
account_id=account_id,
|
||||
primary_region=primary_region,
|
||||
)
|
||||
self.execute_query(ddl)
|
||||
print("[+] VPC Flow Logs table created with partition projection")
|
||||
|
||||
def create_s3_access_table(self, bucket):
|
||||
"""Create S3 access logs table."""
|
||||
ddl = S3_ACCESS_DDL.format(database=self.database, bucket=bucket)
|
||||
self.execute_query(ddl)
|
||||
print("[+] S3 access logs table created")
|
||||
|
||||
def create_alb_table(self, bucket, account_id, primary_region):
|
||||
"""Create ALB access logs table with partition projection."""
|
||||
ddl = ALB_ACCESS_DDL.format(
|
||||
database=self.database,
|
||||
bucket=bucket,
|
||||
account_id=account_id,
|
||||
primary_region=primary_region,
|
||||
)
|
||||
self.execute_query(ddl)
|
||||
print("[+] ALB access logs table created with partition projection")
|
||||
|
||||
def run_forensic_query(self, query_name, start_date, end_date):
|
||||
"""Run a named forensic query with date range."""
|
||||
if query_name not in FORENSIC_QUERIES:
|
||||
available = ", ".join(FORENSIC_QUERIES.keys())
|
||||
print(f"[-] Unknown query: {query_name}. Available: {available}")
|
||||
return None
|
||||
|
||||
query = FORENSIC_QUERIES[query_name].format(
|
||||
database=self.database,
|
||||
start_date=start_date.replace("-", "/"),
|
||||
end_date=end_date.replace("-", "/"),
|
||||
)
|
||||
print(f"[+] Running forensic query: {query_name}")
|
||||
return self.execute_query(query)
|
||||
|
||||
def run_full_investigation(self, start_date, end_date):
|
||||
"""Run all forensic queries and compile a comprehensive report."""
|
||||
report = {
|
||||
"investigation_id": f"inv-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}",
|
||||
"generated_at": datetime.utcnow().isoformat(),
|
||||
"date_range": {"start": start_date, "end": end_date},
|
||||
"findings": {},
|
||||
"summary": {},
|
||||
}
|
||||
|
||||
query_categories = {
|
||||
"access_control": [
|
||||
"unauthorized_access",
|
||||
"privilege_escalation",
|
||||
"console_login_anomalies",
|
||||
],
|
||||
"data_security": [
|
||||
"data_exfiltration_s3",
|
||||
"s3_bulk_download",
|
||||
],
|
||||
"network_activity": [
|
||||
"lateral_movement_vpc",
|
||||
"port_scanning",
|
||||
],
|
||||
"web_attacks": [
|
||||
"alb_injection_attempts",
|
||||
],
|
||||
"infrastructure_changes": [
|
||||
"security_group_changes",
|
||||
],
|
||||
"correlation": [
|
||||
"cross_log_correlation",
|
||||
],
|
||||
}
|
||||
|
||||
total_findings = 0
|
||||
for category, queries in query_categories.items():
|
||||
report["findings"][category] = {}
|
||||
for query_name in queries:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[*] Category: {category} | Query: {query_name}")
|
||||
print(f"{'='*60}")
|
||||
results = self.run_forensic_query(query_name, start_date, end_date)
|
||||
if results is not None:
|
||||
report["findings"][category][query_name] = results
|
||||
total_findings += len(results)
|
||||
print(f"[+] Found {len(results)} results")
|
||||
else:
|
||||
report["findings"][category][query_name] = []
|
||||
print("[!] Query returned no results or failed")
|
||||
|
||||
report["summary"] = {
|
||||
"total_findings": total_findings,
|
||||
"categories_analyzed": len(query_categories),
|
||||
"queries_executed": sum(len(v) for v in query_categories.values()),
|
||||
}
|
||||
|
||||
# Generate severity assessment
|
||||
critical_indicators = []
|
||||
if report["findings"].get("access_control", {}).get("privilege_escalation"):
|
||||
critical_indicators.append("Privilege escalation activity detected")
|
||||
if report["findings"].get("data_security", {}).get("data_exfiltration_s3"):
|
||||
critical_indicators.append("Potential S3 data exfiltration detected")
|
||||
if report["findings"].get("network_activity", {}).get("lateral_movement_vpc"):
|
||||
critical_indicators.append("Lateral movement patterns detected in VPC")
|
||||
if report["findings"].get("correlation", {}).get("cross_log_correlation"):
|
||||
critical_indicators.append(
|
||||
"Cross-log correlation confirms suspicious activity"
|
||||
)
|
||||
|
||||
report["summary"]["critical_indicators"] = critical_indicators
|
||||
report["summary"]["overall_severity"] = (
|
||||
"CRITICAL"
|
||||
if len(critical_indicators) >= 3
|
||||
else "HIGH"
|
||||
if len(critical_indicators) >= 2
|
||||
else "MEDIUM"
|
||||
if len(critical_indicators) >= 1
|
||||
else "LOW"
|
||||
)
|
||||
|
||||
return report
|
||||
|
||||
def generate_timeline(self, start_date, end_date):
|
||||
"""Generate a forensic timeline from all log sources."""
|
||||
timeline_query = """
|
||||
SELECT
|
||||
eventtime AS timestamp,
|
||||
'cloudtrail' AS source,
|
||||
eventsource || ':' || eventname AS event,
|
||||
useridentity.arn AS actor,
|
||||
sourceipaddress AS source_ip,
|
||||
errorcode
|
||||
FROM {database}.cloudtrail_logs
|
||||
WHERE timestamp BETWEEN '{start_date}' AND '{end_date}'
|
||||
AND (
|
||||
errorcode IS NOT NULL
|
||||
OR eventname IN (
|
||||
'ConsoleLogin', 'AssumeRole', 'CreateAccessKey',
|
||||
'AttachUserPolicy', 'AttachRolePolicy',
|
||||
'RunInstances', 'StopInstances', 'TerminateInstances',
|
||||
'CreateBucket', 'DeleteBucket', 'PutBucketPolicy'
|
||||
)
|
||||
)
|
||||
ORDER BY eventtime ASC
|
||||
LIMIT 5000
|
||||
""".format(
|
||||
database=self.database,
|
||||
start_date=start_date.replace("-", "/"),
|
||||
end_date=end_date.replace("-", "/"),
|
||||
)
|
||||
print("[+] Generating forensic timeline...")
|
||||
return self.execute_query(timeline_query)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="AWS Athena Cloud Log Forensics Agent"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--action",
|
||||
required=True,
|
||||
choices=[
|
||||
"setup_tables",
|
||||
"unauthorized_access",
|
||||
"privilege_escalation",
|
||||
"data_exfiltration_s3",
|
||||
"lateral_movement_vpc",
|
||||
"port_scanning",
|
||||
"s3_bulk_download",
|
||||
"alb_injection_attempts",
|
||||
"console_login_anomalies",
|
||||
"security_group_changes",
|
||||
"cross_log_correlation",
|
||||
"full_investigation",
|
||||
"timeline",
|
||||
],
|
||||
)
|
||||
parser.add_argument("--database", default="cloud_forensics")
|
||||
parser.add_argument(
|
||||
"--output-location",
|
||||
default="s3://athena-forensics-output/results/",
|
||||
help="S3 location for Athena query results",
|
||||
)
|
||||
parser.add_argument("--start-date", help="Start date YYYY-MM-DD")
|
||||
parser.add_argument("--end-date", help="End date YYYY-MM-DD")
|
||||
parser.add_argument("--output", default="forensics_report.json")
|
||||
parser.add_argument("--region", default="us-east-1")
|
||||
|
||||
# Table setup arguments
|
||||
parser.add_argument("--cloudtrail-bucket", help="CloudTrail S3 bucket name")
|
||||
parser.add_argument("--vpc-flow-bucket", help="VPC Flow Logs S3 bucket name")
|
||||
parser.add_argument("--s3-access-bucket", help="S3 access logs bucket name")
|
||||
parser.add_argument("--alb-bucket", help="ALB access logs bucket name")
|
||||
parser.add_argument("--account-id", help="AWS account ID")
|
||||
parser.add_argument(
|
||||
"--regions",
|
||||
default="us-east-1",
|
||||
help="Comma-separated AWS regions",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.start_date:
|
||||
args.start_date = (datetime.utcnow() - timedelta(days=7)).strftime("%Y-%m-%d")
|
||||
if not args.end_date:
|
||||
args.end_date = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
|
||||
agent = AthenaForensicsAgent(
|
||||
database=args.database,
|
||||
output_location=args.output_location,
|
||||
region=args.region,
|
||||
)
|
||||
|
||||
if args.action == "setup_tables":
|
||||
if not args.account_id:
|
||||
print("[-] --account-id is required for table setup")
|
||||
return
|
||||
agent.setup_database()
|
||||
regions_list = args.regions.split(",")
|
||||
primary_region = regions_list[0]
|
||||
|
||||
if args.cloudtrail_bucket:
|
||||
agent.create_cloudtrail_table(
|
||||
args.cloudtrail_bucket, args.account_id, regions_list
|
||||
)
|
||||
if args.vpc_flow_bucket:
|
||||
agent.create_vpc_flow_table(
|
||||
args.vpc_flow_bucket, args.account_id, primary_region
|
||||
)
|
||||
if args.s3_access_bucket:
|
||||
agent.create_s3_access_table(args.s3_access_bucket)
|
||||
if args.alb_bucket:
|
||||
agent.create_alb_table(args.alb_bucket, args.account_id, primary_region)
|
||||
|
||||
print("\n[+] Table setup complete")
|
||||
return
|
||||
|
||||
if args.action == "full_investigation":
|
||||
report = agent.run_full_investigation(args.start_date, args.end_date)
|
||||
elif args.action == "timeline":
|
||||
results = agent.generate_timeline(args.start_date, args.end_date)
|
||||
report = {
|
||||
"investigation_type": "timeline",
|
||||
"generated_at": datetime.utcnow().isoformat(),
|
||||
"date_range": {"start": args.start_date, "end": args.end_date},
|
||||
"timeline_events": results or [],
|
||||
}
|
||||
else:
|
||||
results = agent.run_forensic_query(
|
||||
args.action, args.start_date, args.end_date
|
||||
)
|
||||
report = {
|
||||
"investigation_type": args.action,
|
||||
"generated_at": datetime.utcnow().isoformat(),
|
||||
"date_range": {"start": args.start_date, "end": args.end_date},
|
||||
"findings": results or [],
|
||||
"finding_count": len(results) if results else 0,
|
||||
}
|
||||
|
||||
with open(args.output, "w") as f:
|
||||
json.dump(report, f, indent=2, default=str)
|
||||
print(f"\n[+] Report saved to {args.output}")
|
||||
|
||||
if "summary" in report:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"INVESTIGATION SUMMARY")
|
||||
print(f"{'='*60}")
|
||||
summary = report["summary"]
|
||||
print(f"Total findings: {summary.get('total_findings', 0)}")
|
||||
print(f"Overall severity: {summary.get('overall_severity', 'N/A')}")
|
||||
for indicator in summary.get("critical_indicators", []):
|
||||
print(f" [!] {indicator}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user