diff --git a/config/config.py b/config/config.py index 0ad5db2c..35babb06 100644 --- a/config/config.py +++ b/config/config.py @@ -21,6 +21,7 @@ csv_file_suffix = ".csv" json_file_suffix = ".json" json_asff_file_suffix = ".asff.json" config_yaml = "providers/aws/config.yaml" +allowlist_yaml = "providers/aws/allowlist.yaml" def change_config_var(variable, value): diff --git a/lib/check/check.py b/lib/check/check.py index 921755c7..d02a3e13 100644 --- a/lib/check/check.py +++ b/lib/check/check.py @@ -183,6 +183,7 @@ def set_output_options( input_output_directory: str, security_hub_enabled: bool, output_filename: str, + allowlist_file: str, ): global output_options output_options = Output_From_Options( @@ -191,6 +192,7 @@ def set_output_options( output_directory=input_output_directory, security_hub_enabled=security_hub_enabled, output_filename=output_filename, + allowlist_file=allowlist_file, # set input options here ) return output_options diff --git a/lib/check/models.py b/lib/check/models.py index fb5c9581..8334ed64 100644 --- a/lib/check/models.py +++ b/lib/check/models.py @@ -15,6 +15,7 @@ class Output_From_Options: output_directory: str security_hub_enabled: bool output_filename: str + allowlist_file: str # Testing Pending diff --git a/lib/outputs/outputs.py b/lib/outputs/outputs.py index d470dd3a..931e5b92 100644 --- a/lib/outputs/outputs.py +++ b/lib/outputs/outputs.py @@ -24,12 +24,12 @@ from lib.outputs.models import ( Severity, ) from lib.utils.utils import file_exists, hash_sha512, open_file +from providers.aws.lib.allowlist.allowlist import is_allowlisted from providers.aws.lib.security_hub.security_hub import send_to_security_hub def report(check_findings, output_options, audit_info): check_findings.sort(key=lambda x: x.region) - csv_fields = [] # check output options file_descriptors = {} @@ -46,6 +46,15 @@ def report(check_findings, output_options, audit_info): if check_findings: for finding in check_findings: + # Check if finding is allowlisted + if is_allowlisted( + output_options.allowlist_file, + audit_info.audited_account, + finding.check_metadata.CheckID, + finding.region, + finding.resource_id, + ): + finding.status = "WARNING" # Print findings by stdout color = set_report_color(finding.status) if output_options.is_quiet and "FAIL" in finding.status: diff --git a/providers/aws/allowlist.yaml b/providers/aws/allowlist.yaml new file mode 100644 index 00000000..1b559580 --- /dev/null +++ b/providers/aws/allowlist.yaml @@ -0,0 +1,37 @@ +### Account, Check and/or Region can be * to apply for all the cases +### Resources is a list that can have either Regex or Keywords: +########################### ALLOWLIST EXAMPLE ########################### +Allowlist: + Accounts: + "123456789012": + Checks: + "iam_user_hardware_mfa_enabled": + Regions: + - "us-east-1" + Resources: + - "user-1" # Will ignore user-1 in check iam_user_hardware_mfa_enabled + - "user-2" # Will ignore user-2 in check iam_user_hardware_mfa_enabled + "*": + Regions: + - "*" + Resources: + - "test" # Will ignore every resource containing the string "test" in every account and region + + "098765432109": + Checks: + "s3_bucket_object_versioning": + Regions: + - "eu-west-1" + - "us-east-1" + Resources: + - "ci-logs" # Will ignore bucket "ci-logs" AND ALSO bucket "ci-logs-replica" in specified check and regions + - "logs" # Will ignore EVERY BUCKET containing the string "logs" in specified check and regions + - "[[:alnum:]]+-logs" # Will ignore all buckets containing the terms ci-logs, qa-logs, etc. in specified check and regions + +# EXAMPLE: CONTROL TOWER (to migrate) +# When using Control Tower, guardrails prevent access to certain protected resources. The allowlist +# below ensures that warnings instead of errors are reported for the affected resources. +#extra734:aws-controltower-logs-[[:digit:]]+-[[:alpha:]\-]+ +#extra734:aws-controltower-s3-access-logs-[[:digit:]]+-[[:alpha:]\-]+ +#extra764:aws-controltower-logs-[[:digit:]]+-[[:alpha:]\-]+ +#extra764:aws-controltower-s3-access-logs-[[:digit:]]+-[[:alpha:]\-]+ diff --git a/providers/aws/lib/allowlist/__init__.py b/providers/aws/lib/allowlist/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/aws/lib/allowlist/allowlist.py b/providers/aws/lib/allowlist/allowlist.py new file mode 100644 index 00000000..68b316c4 --- /dev/null +++ b/providers/aws/lib/allowlist/allowlist.py @@ -0,0 +1,105 @@ +import re +import sys + +import yaml +from boto3.dynamodb.conditions import Attr + +from lib.logger import logger + + +def parse_allowlist_file(audit_info, allowlist_file): + try: + # Check if file is a S3 URI + if re.search("^s3://([^/]+)/(.*?([^/]+))$", allowlist_file): + bucket = allowlist_file.split("/")[2] + key = ("/").join(allowlist_file.split("/")[3:]) + s3_client = audit_info.audit_session.client("s3") + allowlist = yaml.safe_load( + s3_client.get_object(Bucket=bucket, Key=key)["Body"] + )["Allowlist"] + # Check if file is a DynamoDB ARN + elif re.search( + "^arn:[aws\|aws\-cn\|aws\-us\-gov]+:dynamodb:[a-z]{2}-[north\|south\|east\|west\|central]+-[1-9]{1}:[0-9]{12}:table\/[a-zA-Z0-9._-]+$", + allowlist_file, + ): + allowlist = {"Accounts": {}} + table_region = allowlist_file.split(":")[3] + dynamodb_resource = audit_info.audit_session.resource( + "dynamodb", region_name=table_region + ) + dynamo_table = dynamodb_resource.Table(allowlist_file.split("/")[1]) + response = dynamo_table.scan( + FilterExpression=Attr("Accounts").is_in( + [audit_info.audited_account, "*"] + ) + ) + dynamodb_items = response["Items"] + # Paginate through all results + while "LastEvaluatedKey" in dynamodb_items: + response = dynamo_table.scan( + ExclusiveStartKey=response["LastEvaluatedKey"], + FilterExpression=Attr("Accounts").is_in( + [audit_info.audited_account, "*"] + ), + ) + dynamodb_items.update(response["Items"]) + for item in dynamodb_items: + # Create allowlist for every item + allowlist["Accounts"][item["Accounts"]] = { + "Checks": { + item["Checks"]: { + "Regions": item["Regions"], + "Resources": item["Resources"], + } + } + } + else: + with open(allowlist_file) as f: + allowlist = yaml.safe_load(f)["Allowlist"] + print(allowlist) + return allowlist + except Exception as error: + logger.critical(f"{error.__class__.__name__} -- {error}") + sys.exit() + + +def is_allowlisted(allowlist, audited_account, check, region, resource): + try: + if audited_account in allowlist["Accounts"]: + if is_allowlisted_in_check(allowlist, audited_account, check, region, resource): + return True + # If there is a *, it affects to all accounts + if "*" in allowlist["Accounts"]: + audited_account = "*" + if is_allowlisted_in_check(allowlist, audited_account, check, region, resource): + return True + return False + except Exception as error: + logger.critical(f"{error.__class__.__name__} -- {error}") + sys.exit() + + +def is_allowlisted_in_check(allowlist, audited_account, check, region, resource): + # If there is a *, it affects to all checks + if "*" in allowlist["Accounts"][audited_account]["Checks"]: + check = "*" + if is_allowlisted_in_region(allowlist, audited_account, check, region, resource): + return True + # Check if there is the specific check + if check in allowlist["Accounts"][audited_account]["Checks"]: + if is_allowlisted_in_region(allowlist, audited_account, check, region, resource): + return True + return False + + +def is_allowlisted_in_region(allowlist, audited_account, check, region, resource): + # If there is a *, it affects to all regions + if "*" in allowlist["Accounts"][audited_account]["Checks"][check]["Regions"]: + for elem in allowlist["Accounts"][audited_account]["Checks"][check]["Resources"]: + if re.search(elem, resource): + return True + # Check if there is the specific region + if region in allowlist["Accounts"][audited_account]["Checks"][check]["Regions"]: + for elem in allowlist["Accounts"][audited_account]["Checks"][check]["Resources"]: + if re.search(elem, resource): + return True diff --git a/providers/aws/lib/allowlist/allowlist_test.py b/providers/aws/lib/allowlist/allowlist_test.py new file mode 100644 index 00000000..ea512be6 --- /dev/null +++ b/providers/aws/lib/allowlist/allowlist_test.py @@ -0,0 +1,142 @@ +import yaml +from boto3 import resource, session +from moto import mock_dynamodb, mock_s3 + +from providers.aws.lib.allowlist.allowlist import is_allowlisted, parse_allowlist_file +from providers.aws.lib.audit_info.models import AWS_Audit_Info + +AWS_ACCOUNT_NUMBER = 123456789012 +AWS_REGION = "us-east-1" + + +class Test_Allowlist: + + # Mocked Audit Info + def set_mocked_audit_info(self): + audit_info = AWS_Audit_Info( + original_session=None, + audit_session=session.Session( + profile_name=None, + botocore_session=None, + ), + audited_account=AWS_ACCOUNT_NUMBER, + audited_user_id=None, + audited_partition="aws", + audited_identity_arn=None, + profile=None, + profile_region=None, + credentials=None, + assumed_role_info=None, + audited_regions=None, + organizations_metadata=None, + ) + return audit_info + + # Test S3 allowlist + @mock_s3 + def test_s3_allowlist(self): + audit_info = self.set_mocked_audit_info() + # Create bucket and upload allowlist yaml + s3_resource = resource("s3", region_name=AWS_REGION) + s3_resource.create_bucket(Bucket="test-allowlist") + s3_resource.Object("test-allowlist", "allowlist.yaml").put( + Body=open( + "providers/aws/lib/allowlist/fixtures/allowlist.yaml", + "rb", + ) + ) + + with open("providers/aws/lib/allowlist/fixtures/allowlist.yaml") as f: + assert yaml.safe_load(f)["Allowlist"] == parse_allowlist_file( + audit_info, "s3://test-allowlist/allowlist.yaml" + ) + + # Test S3 allowlist + @mock_dynamodb + def test_dynamo_allowlist(self): + audit_info = self.set_mocked_audit_info() + # Create table and put item + dynamodb_resource = resource("dynamodb", region_name=AWS_REGION) + table_name = "test-allowlist" + params = { + "TableName": table_name, + "KeySchema": [ + {"AttributeName": "Accounts", "KeyType": "HASH"}, + {"AttributeName": "Checks", "KeyType": "RANGE"}, + ], + "AttributeDefinitions": [ + {"AttributeName": "Accounts", "AttributeType": "S"}, + {"AttributeName": "Checks", "AttributeType": "S"}, + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": 10, + "WriteCapacityUnits": 10, + }, + } + table = dynamodb_resource.create_table(**params) + table.put_item( + Item={ + "Accounts": "*", + "Checks": "iam_user_hardware_mfa_enabled", + "Regions": ["eu-west-1", "us-east-1"], + "Resources": ["keyword"], + } + ) + + assert ( + "keyword" + in parse_allowlist_file( + audit_info, + "arn:aws:dynamodb:" + + AWS_REGION + + ":" + + str(AWS_ACCOUNT_NUMBER) + + ":table/" + + table_name, + )["Accounts"]["*"]["Checks"]["iam_user_hardware_mfa_enabled"]["Resources"] + ) + + # Allowlist checks + def test_is_allowlisted(self): + + # Allowlist example + allowlist = { + "Accounts": { + "*": { + "Checks": { + "check_test": { + "Regions": ["us-east-1", "eu-west-1"], + "Resources": ["prowler", "^test"], + } + } + } + } + } + + assert ( + is_allowlisted( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler" + ) + == True + ) + + assert ( + is_allowlisted( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-test" + ) + == True + ) + + assert ( + is_allowlisted( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "test-prowler" + ) + == True + ) + + assert ( + is_allowlisted( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test" + ) + == False + ) diff --git a/providers/aws/lib/allowlist/fixtures/allowlist.yaml b/providers/aws/lib/allowlist/fixtures/allowlist.yaml new file mode 100644 index 00000000..1b559580 --- /dev/null +++ b/providers/aws/lib/allowlist/fixtures/allowlist.yaml @@ -0,0 +1,37 @@ +### Account, Check and/or Region can be * to apply for all the cases +### Resources is a list that can have either Regex or Keywords: +########################### ALLOWLIST EXAMPLE ########################### +Allowlist: + Accounts: + "123456789012": + Checks: + "iam_user_hardware_mfa_enabled": + Regions: + - "us-east-1" + Resources: + - "user-1" # Will ignore user-1 in check iam_user_hardware_mfa_enabled + - "user-2" # Will ignore user-2 in check iam_user_hardware_mfa_enabled + "*": + Regions: + - "*" + Resources: + - "test" # Will ignore every resource containing the string "test" in every account and region + + "098765432109": + Checks: + "s3_bucket_object_versioning": + Regions: + - "eu-west-1" + - "us-east-1" + Resources: + - "ci-logs" # Will ignore bucket "ci-logs" AND ALSO bucket "ci-logs-replica" in specified check and regions + - "logs" # Will ignore EVERY BUCKET containing the string "logs" in specified check and regions + - "[[:alnum:]]+-logs" # Will ignore all buckets containing the terms ci-logs, qa-logs, etc. in specified check and regions + +# EXAMPLE: CONTROL TOWER (to migrate) +# When using Control Tower, guardrails prevent access to certain protected resources. The allowlist +# below ensures that warnings instead of errors are reported for the affected resources. +#extra734:aws-controltower-logs-[[:digit:]]+-[[:alpha:]\-]+ +#extra734:aws-controltower-s3-access-logs-[[:digit:]]+-[[:alpha:]\-]+ +#extra764:aws-controltower-logs-[[:digit:]]+-[[:alpha:]\-]+ +#extra764:aws-controltower-s3-access-logs-[[:digit:]]+-[[:alpha:]\-]+ diff --git a/prowler b/prowler old mode 100755 new mode 100644 index 74defd40..523cd1b9 --- a/prowler +++ b/prowler @@ -7,6 +7,7 @@ from os import mkdir from os.path import isdir from config.config import ( + allowlist_yaml, change_config_var, default_output_directory, output_file_timestamp, @@ -29,6 +30,7 @@ from lib.check.checks_loader import load_checks_to_execute from lib.logger import logger, set_logging_config from lib.outputs.outputs import close_json, send_to_s3_bucket from providers.aws.aws_provider import provider_set_session +from providers.aws.lib.allowlist.allowlist import parse_allowlist_file from providers.aws.lib.security_hub.security_hub import ( resolve_security_hub_previous_findings, ) @@ -186,6 +188,12 @@ if __name__ == "__main__": default=None, help="Shodan API key used by check ec2_elastic_ip_shodan.", ) + parser.add_argument( + "-w", + "--allowlist-file", + nargs="?", + help="Path for allowlist yaml file, by default is 'providers/aws/allowlist.yaml'. See default yaml for reference and format.", + ) # Parse Arguments args = parser.parse_args() @@ -299,9 +307,18 @@ if __name__ == "__main__": f"prowler-output-{audit_info.audited_account}-{output_file_timestamp}" ) + # Parse content from Allowlist file and get it, if necessary, from S3 + if args.allowlist_file: + allowlist_file = parse_allowlist_file(audit_info, args.allowlist_file) + # Setting output options audit_output_options = set_output_options( - args.quiet, output_modes, output_directory, args.security_hub, output_filename + args.quiet, + output_modes, + output_directory, + args.security_hub, + output_filename, + allowlist_file, ) # Execute checks