From da76f69e51b3b33d278bcf254dd5bd81dcf3139c Mon Sep 17 00:00:00 2001 From: Sergio Garcia <38561120+sergargar@users.noreply.github.com> Date: Mon, 25 Jul 2022 13:45:31 -0400 Subject: [PATCH] feat(s3_checks): add several checks for s3 (#1266) * feat(checks): add extra718 * feat(checks): add extra763 Co-authored-by: sergargar --- providers/aws/aws_provider.py | 119 +++--------------- providers/aws/services/ec2/ec2_service.py | 31 +---- providers/aws/services/s3/__init__.py | 0 .../s3_bucket_object_versioning/__init__.py | 0 .../s3_bucket_object_versioning.metadata.json | 37 ++++++ .../s3_bucket_object_versioning.py | 33 +++++ .../__init__.py | 0 ...erver_access_logging_enabled.metadata.json | 37 ++++++ ...s3_bucket_server_access_logging_enabled.py | 29 +++++ providers/aws/services/s3/s3_service.py | 100 +++++++++++++++ 10 files changed, 257 insertions(+), 129 deletions(-) create mode 100644 providers/aws/services/s3/__init__.py create mode 100644 providers/aws/services/s3/s3_bucket_object_versioning/__init__.py create mode 100644 providers/aws/services/s3/s3_bucket_object_versioning/s3_bucket_object_versioning.metadata.json create mode 100644 providers/aws/services/s3/s3_bucket_object_versioning/s3_bucket_object_versioning.py create mode 100644 providers/aws/services/s3/s3_bucket_server_access_logging_enabled/__init__.py create mode 100644 providers/aws/services/s3/s3_bucket_server_access_logging_enabled/s3_bucket_server_access_logging_enabled.metadata.json create mode 100644 providers/aws/services/s3/s3_bucket_server_access_logging_enabled/s3_bucket_server_access_logging_enabled.py create mode 100644 providers/aws/services/s3/s3_service.py diff --git a/providers/aws/aws_provider.py b/providers/aws/aws_provider.py index 55769f5f..f121463d 100644 --- a/providers/aws/aws_provider.py +++ b/providers/aws/aws_provider.py @@ -8,10 +8,10 @@ from boto3 import client, session from botocore.credentials import RefreshableCredentials from botocore.session import get_session -from config.config import json_asff_file_suffix, timestamp_utc +from config.config import aws_services_json_file from lib.arn.arn import arn_parsing from lib.logger import logger -from lib.outputs.models import Check_Output_JSON_ASFF +from lib.utils.utils import open_file, parse_json_file from providers.aws.models import ( AWS_Assume_Role, AWS_Audit_Info, @@ -278,102 +278,21 @@ def get_organizations_metadata( return organizations_info -def send_to_security_hub( - region: str, finding_output: Check_Output_JSON_ASFF, session: session.Session -): - try: - logger.info("Sending findings to Security Hub.") - # Check if security hub is enabled in current region - security_hub_client = session.client("securityhub", region_name=region) - security_hub_client.describe_hub() +def generate_regional_clients(service, audit_info): + regional_clients = [] + # Get json locally + f = open_file(aws_services_json_file) + data = parse_json_file(f) + json_regions = data["services"][service]["regions"][audit_info.audited_partition] + if audit_info.audited_regions: # Check for input aws audit_info.audited_regions + regions = list( + set(json_regions).intersection(audit_info.audited_regions) + ) # Get common regions between input and json + else: # Get all regions from json of the service and partition + regions = json_regions + for region in regions: + regional_client = audit_info.audit_session.client(service, region_name=region) + regional_client.region = region + regional_clients.append(regional_client) - # Check if Prowler integration is enabled in Security Hub - if "prowler/prowler" not in str( - security_hub_client.list_enabled_products_for_import() - ): - logger.error( - f"Security Hub is enabled in {region} but Prowler integration does not accept findings. More info: https://github.com/prowler-cloud/prowler/#security-hub-integration" - ) - - # Send finding to Security Hub - batch_import = security_hub_client.batch_import_findings( - Findings=[finding_output.dict()] - ) - if batch_import["FailedCount"] > 0: - failed_import = batch_import["FailedFindings"][0] - logger.error( - f"Failed to send archived findings to AWS Security Hub -- {failed_import['ErrorCode']} -- {failed_import['ErrorMessage']}" - ) - - except Exception as error: - logger.error(f"{error.__class__.__name__} -- {error} in region {region}") - - -# Move previous Security Hub check findings to ARCHIVED (as prowler didn't re-detect them) -def resolve_security_hub_previous_findings( - output_directory: str, audit_info: AWS_Audit_Info -) -> list: - logger.info("Checking previous findings in Security Hub to archive them.") - # Read current findings from json-asff file - with open( - f"{output_directory}/prowler-output-{audit_info.audited_account}-{json_asff_file_suffix}" - ) as f: - json_asff_file = json.load(f) - - # Sort by region - json_asff_file = sorted(json_asff_file, key=itemgetter("ProductArn")) - # Group by region - for product_arn, current_findings in groupby( - json_asff_file, key=itemgetter("ProductArn") - ): - region = product_arn.split(":")[3] - try: - # Check if security hub is enabled in current region - security_hub_client = audit_info.audit_session.client( - "securityhub", region_name=region - ) - security_hub_client.describe_hub() - # Get current findings IDs - current_findings_ids = [] - for finding in current_findings: - current_findings_ids.append(finding["Id"]) - # Get findings of that region - security_hub_client = audit_info.audit_session.client( - "securityhub", region_name=region - ) - findings_filter = { - "ProductName": [{"Value": "Prowler", "Comparison": "EQUALS"}], - "RecordState": [{"Value": "ACTIVE", "Comparison": "EQUALS"}], - "AwsAccountId": [ - {"Value": audit_info.audited_account, "Comparison": "EQUALS"} - ], - "Region": [{"Value": region, "Comparison": "EQUALS"}], - } - get_findings_paginator = security_hub_client.get_paginator("get_findings") - findings_to_archive = [] - for page in get_findings_paginator.paginate(Filters=findings_filter): - # Archive findings that have not appear in this execution - for finding in page["Findings"]: - if finding["Id"] not in current_findings_ids: - finding["RecordState"] = "ARCHIVED" - finding["UpdatedAt"] = timestamp_utc.strftime( - "%Y-%m-%dT%H:%M:%SZ" - ) - findings_to_archive.append(finding) - logger.info(f"Archiving {len(findings_to_archive)} findings.") - # Send archive findings to SHub - list_chunked = [ - findings_to_archive[i : i + 100] - for i in range(0, len(findings_to_archive), 100) - ] - for findings in list_chunked: - batch_import = security_hub_client.batch_import_findings( - Findings=findings - ) - if batch_import["FailedCount"] > 0: - failed_import = batch_import["FailedFindings"][0] - logger.error( - f"Failed to send archived findings to AWS Security Hub -- {failed_import['ErrorCode']} -- {failed_import['ErrorMessage']}" - ) - except Exception as error: - logger.error(f"{error.__class__.__name__} -- {error} in region {region}") + return regional_clients diff --git a/providers/aws/services/ec2/ec2_service.py b/providers/aws/services/ec2/ec2_service.py index ea938931..bf4b242d 100644 --- a/providers/aws/services/ec2/ec2_service.py +++ b/providers/aws/services/ec2/ec2_service.py @@ -1,9 +1,7 @@ import threading -from config.config import aws_services_json_file from lib.logger import logger -from lib.utils.utils import open_file, parse_json_file -from providers.aws.aws_provider import current_audit_info +from providers.aws.aws_provider import current_audit_info, generate_regional_clients ################## EC2 @@ -12,37 +10,12 @@ class EC2: self.service = "ec2" self.session = audit_info.audit_session self.audited_account = audit_info.audited_account - self.regional_clients = self.__generate_regional_clients__( - self.service, audit_info - ) + self.regional_clients = generate_regional_clients(self.service, audit_info) self.__threading_call__(self.__describe_snapshots__) def __get_session__(self): return self.session - def __generate_regional_clients__(self, service, audit_info): - regional_clients = [] - # Get json locally - f = open_file(aws_services_json_file) - data = parse_json_file(f) - json_regions = data["services"][service]["regions"][ - audit_info.audited_partition - ] - if audit_info.audited_regions: # Check for input aws audit_info.audited_regions - regions = list( - set(json_regions).intersection(audit_info.audited_regions) - ) # Get common regions between input and json - else: # Get all regions from json of the service and partition - regions = json_regions - for region in regions: - regional_client = audit_info.audit_session.client( - service, region_name=region - ) - regional_client.region = region - regional_clients.append(regional_client) - - return regional_clients - def __threading_call__(self, call): threads = [] for regional_client in self.regional_clients: diff --git a/providers/aws/services/s3/__init__.py b/providers/aws/services/s3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/aws/services/s3/s3_bucket_object_versioning/__init__.py b/providers/aws/services/s3/s3_bucket_object_versioning/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/aws/services/s3/s3_bucket_object_versioning/s3_bucket_object_versioning.metadata.json b/providers/aws/services/s3/s3_bucket_object_versioning/s3_bucket_object_versioning.metadata.json new file mode 100644 index 00000000..3e091ae0 --- /dev/null +++ b/providers/aws/services/s3/s3_bucket_object_versioning/s3_bucket_object_versioning.metadata.json @@ -0,0 +1,37 @@ +{ + "Categories": [], + "CheckAlias": "extra763", + "CheckID": "s3_bucket_object_versioning", + "CheckName": "s3_bucket_object_versioning", + "CheckTitle": "Check if S3 buckets have object versioning enabled", + "CheckType": "Data Protection", + "Compliance": [], + "DependsOn": [], + "Description": "Check if S3 buckets have object versioning enabled", + "Notes": "", + "Provider": "aws", + "RelatedTo": [], + "RelatedUrl": "", + "Remediation": { + "Code": { + "CLI": "", + "NativeIaC": "", + "Other": "", + "Terraform": "" + }, + "Recommendation": { + "Text": "Configure versioning using the Amazon console or API for buckets with sensitive information that is changing frecuently; and backup may not be enough to capture all the changes.", + "Url": "https://docs.aws.amazon.com/AmazonS3/latest/dev-retired/Versioning.html" + } + }, + "ResourceIdTemplate": "arn:partition:service:region:account-id:resource-id", + "ResourceType": "AwsS3Bucket", + "Risk": "With versioning, you can easily recover from both unintended user actions and application failures.", + "ServiceName": "s3", + "Severity": "medium", + "SubServiceName": "", + "Tags": { + "Tag1Key": "value", + "Tag2Key": "value" + } +} diff --git a/providers/aws/services/s3/s3_bucket_object_versioning/s3_bucket_object_versioning.py b/providers/aws/services/s3/s3_bucket_object_versioning/s3_bucket_object_versioning.py new file mode 100644 index 00000000..17276cb4 --- /dev/null +++ b/providers/aws/services/s3/s3_bucket_object_versioning/s3_bucket_object_versioning.py @@ -0,0 +1,33 @@ +from lib.check.models import Check, Check_Report +from providers.aws.services.s3.s3_service import s3_client + + +class s3_bucket_object_versioning(Check): + def execute(self): + findings = [] + for regional_client in s3_client.regional_clients: + region = regional_client.region + if regional_client.buckets: + for bucket in regional_client.buckets: + report = Check_Report(self.metadata) + report.region = region + report.resource_id = bucket.name + if bucket.versioning: + report.status = "PASS" + report.status_extended = ( + f"S3 Bucket {bucket.name} has versioning enabled." + ) + else: + report.status = "FAIL" + report.status_extended = ( + f"S3 Bucket {bucket.name} has versioning disabled." + ) + findings.append(report) + else: + report = Check_Report(self.metadata) + report.status = "PASS" + report.status_extended = "There are no S3 buckets." + report.region = region + findings.append(report) + + return findings diff --git a/providers/aws/services/s3/s3_bucket_server_access_logging_enabled/__init__.py b/providers/aws/services/s3/s3_bucket_server_access_logging_enabled/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/aws/services/s3/s3_bucket_server_access_logging_enabled/s3_bucket_server_access_logging_enabled.metadata.json b/providers/aws/services/s3/s3_bucket_server_access_logging_enabled/s3_bucket_server_access_logging_enabled.metadata.json new file mode 100644 index 00000000..904abad0 --- /dev/null +++ b/providers/aws/services/s3/s3_bucket_server_access_logging_enabled/s3_bucket_server_access_logging_enabled.metadata.json @@ -0,0 +1,37 @@ +{ + "Categories": [], + "CheckAlias": "extra718", + "CheckID": "s3_bucket_server_access_logging_enabled", + "CheckName": "s3_bucket_server_access_logging_enabled", + "CheckTitle": "Check if S3 buckets have server access logging enabled", + "CheckType": "Logging and Monitoring", + "Compliance": [], + "DependsOn": [], + "Description": "Check if S3 buckets have server access logging enabled", + "Notes": "", + "Provider": "aws", + "RelatedTo": [], + "RelatedUrl": "", + "Remediation": { + "Code": { + "CLI": "", + "NativeIaC": "", + "Other": "", + "Terraform": "" + }, + "Recommendation": { + "Text": "Ensure that S3 buckets have Logging enabled. CloudTrail data events can be used in place of S3 bucket logging. If that is the case, this finding can be considered a false positive.", + "Url": "https://docs.aws.amazon.com/AmazonS3/latest/dev/security-best-practices.html" + } + }, + "ResourceIdTemplate": "arn:partition:service:region:account-id:resource-id", + "ResourceType": "AwsS3Bucket", + "Risk": "Server access logs can assist you in security and access audits; help you learn about your customer base; and understand your Amazon S3 bill.", + "ServiceName": "s3", + "Severity": "medium", + "SubServiceName": "", + "Tags": { + "Tag1Key": "value", + "Tag2Key": "value" + } +} diff --git a/providers/aws/services/s3/s3_bucket_server_access_logging_enabled/s3_bucket_server_access_logging_enabled.py b/providers/aws/services/s3/s3_bucket_server_access_logging_enabled/s3_bucket_server_access_logging_enabled.py new file mode 100644 index 00000000..c21a8480 --- /dev/null +++ b/providers/aws/services/s3/s3_bucket_server_access_logging_enabled/s3_bucket_server_access_logging_enabled.py @@ -0,0 +1,29 @@ +from lib.check.models import Check, Check_Report +from providers.aws.services.s3.s3_service import s3_client + + +class s3_bucket_server_access_logging_enabled(Check): + def execute(self): + findings = [] + for regional_client in s3_client.regional_clients: + region = regional_client.region + if regional_client.buckets: + for bucket in regional_client.buckets: + report = Check_Report(self.metadata) + report.region = region + report.resource_id = bucket.name + if bucket.logging: + report.status = "PASS" + report.status_extended = f"S3 Bucket {bucket.name} has server access logging enabled." + else: + report.status = "FAIL" + report.status_extended = f"S3 Bucket {bucket.name} has server access logging disabled." + findings.append(report) + else: + report = Check_Report(self.metadata) + report.status = "PASS" + report.status_extended = "There are no S3 buckets." + report.region = region + findings.append(report) + + return findings diff --git a/providers/aws/services/s3/s3_service.py b/providers/aws/services/s3/s3_service.py new file mode 100644 index 00000000..7d5ebb80 --- /dev/null +++ b/providers/aws/services/s3/s3_service.py @@ -0,0 +1,100 @@ +import threading +from dataclasses import dataclass + +from lib.logger import logger +from providers.aws.aws_provider import current_audit_info, generate_regional_clients + + +################## S3 +class S3: + def __init__(self, audit_info): + self.service = "s3" + self.session = audit_info.audit_session + self.audited_account = audit_info.audited_account + self.regional_clients = generate_regional_clients(self.service, audit_info) + self.__threading_call__(self.__list_buckets__) + self.__threading_call__(self.__get_bucket_versioning__) + self.__threading_call__(self.__get_bucket_logging__) + + def __get_session__(self): + return self.session + + def __threading_call__(self, call): + threads = [] + for regional_client in self.regional_clients: + threads.append(threading.Thread(target=call, args=(regional_client,))) + for t in threads: + t.start() + for t in threads: + t.join() + + def __list_buckets__(self, regional_client): + logger.info("S3 - Listing buckets...") + try: + list_buckets = regional_client.list_buckets() + buckets = [] + for bucket in list_buckets["Buckets"]: + try: + bucket_region = regional_client.get_bucket_location( + Bucket=bucket["Name"] + )["LocationConstraint"] + if regional_client.region == bucket_region or ( + regional_client.region == "us-east-1" and not bucket_region + ): # If us-east-1, bucket_region is none + buckets.append(Bucket(bucket["Name"])) + except Exception as error: + if error.__class__.__name__ != "NoSuchBucket": + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}: {error}" + ) + regional_client.buckets = buckets + except Exception as error: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}: {error}" + ) + + def __get_bucket_versioning__(self, regional_client): + logger.info("S3 - Get buckets versioning...") + try: + if hasattr(regional_client, "buckets"): + for bucket in regional_client.buckets: + bucket_versioning = regional_client.get_bucket_versioning( + Bucket=bucket.name + ) + if "Status" in bucket_versioning: + if "Enabled" == bucket_versioning["Status"]: + bucket.versioning = True + except Exception as error: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}: {error}" + ) + + def __get_bucket_logging__(self, regional_client): + logger.info("S3 - Get buckets logging...") + try: + if hasattr(regional_client, "buckets"): + for bucket in regional_client.buckets: + bucket_logging = regional_client.get_bucket_logging( + Bucket=bucket.name + ) + if "LoggingEnabled" in bucket_logging: + bucket.logging = True + except Exception as error: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}: {error}" + ) + + +@dataclass +class Bucket: + name: str + versioning: bool + logging: bool + + def __init__(self, name): + self.name = name + self.versioning = False + self.logging = False + + +s3_client = S3(current_audit_info)