From 51eacbfac54f46eea11940fc82db89cdd4dcd0a3 Mon Sep 17 00:00:00 2001 From: Sergio Garcia <38561120+sergargar@users.noreply.github.com> Date: Tue, 21 Mar 2023 11:14:59 +0100 Subject: [PATCH] feat(allowlist): add tags filter to allowlist (#2105) --- docs/tutorials/allowlist.md | 34 ++- prowler/config/allowlist.yaml | 16 +- prowler/lib/outputs/models.py | 2 +- prowler/lib/outputs/outputs.py | 2 + .../providers/aws/lib/allowlist/allowlist.py | 94 ++++++-- .../aws/lib/allowlist/allowlist_test.py | 227 ++++++++++++++++-- 6 files changed, 320 insertions(+), 55 deletions(-) diff --git a/docs/tutorials/allowlist.md b/docs/tutorials/allowlist.md index 33fe0cb2..d864bb43 100644 --- a/docs/tutorials/allowlist.md +++ b/docs/tutorials/allowlist.md @@ -8,34 +8,45 @@ You can use `-w`/`--allowlist-file` with the path of your allowlist yaml file, b ## Allowlist Yaml File Syntax ### Account, Check and/or Region can be * to apply for all the cases - ### Resources is a list that can have either Regex or Keywords: + ### Resources is a list that can have either Regex or Keywords + ### Tags is an optional list containing tuples of 'key=value' ########################### ALLOWLIST EXAMPLE ########################### Allowlist: Accounts: "123456789012": - Checks: + Checks: "iam_user_hardware_mfa_enabled": - Regions: + Regions: - "us-east-1" - Resources: + Resources: - "user-1" # Will ignore user-1 in check iam_user_hardware_mfa_enabled - "user-2" # Will ignore user-2 in check iam_user_hardware_mfa_enabled "*": - Regions: + Regions: - "*" - Resources: - - "test" # Will ignore every resource containing the string "test" in every account and region + Resources: + - "test" # Will ignore every resource containing the string "test" and the tags 'test=test' and 'project=test' in account 123456789012 and every region + Tags: + - "test=test" # Will ignore every resource containing the string "test" and the tags 'test=test' and 'project=test' in account 123456789012 and every region + - "project=test" "*": - Checks: + Checks: "s3_bucket_object_versioning": - Regions: + Regions: - "eu-west-1" - "us-east-1" - Resources: + Resources: - "ci-logs" # Will ignore bucket "ci-logs" AND ALSO bucket "ci-logs-replica" in specified check and regions - "logs" # Will ignore EVERY BUCKET containing the string "logs" in specified check and regions - "[[:alnum:]]+-logs" # Will ignore all buckets containing the terms ci-logs, qa-logs, etc. in specified check and regions + "*": + Regions: + - "*" + Resources: + - "*" + Tags: + - "environment=dev" # Will ignore every resource containing the tag 'environment=dev' in every account and region ## Supported Allowlist Locations @@ -70,6 +81,7 @@ prowler aws -w arn:aws:dynamodb:::table/ - Checks (String): This field can contain either a Prowler Check Name or an `*` (which applies to all the scanned checks). - Regions (List): This field contains a list of regions where this allowlist rule is applied (it can also contains an `*` to apply all scanned regions). - Resources (List): This field contains a list of regex expressions that applies to the resources that are wanted to be allowlisted. + - Tags (List): -Optional- This field contains a list of tuples in the form of 'key=value' that applies to the resources tags that are wanted to be allowlisted. @@ -101,7 +113,7 @@ generates an Allowlist: ``` def handler(event, context): checks = {} - checks["vpc_flow_logs_enabled"] = { "Regions": [ "*" ], "Resources": [ "" ] } + checks["vpc_flow_logs_enabled"] = { "Regions": [ "*" ], "Resources": [ "" ], Optional("Tags"): [ "key:value" ] } al = { "Allowlist": { "Accounts": { "*": { "Checks": checks } } } } return al diff --git a/prowler/config/allowlist.yaml b/prowler/config/allowlist.yaml index e83e571f..6c8f9512 100644 --- a/prowler/config/allowlist.yaml +++ b/prowler/config/allowlist.yaml @@ -1,5 +1,6 @@ ### Account, Check and/or Region can be * to apply for all the cases -### Resources is a list that can have either Regex or Keywords: +### Resources is a list that can have either Regex or Keywords +### Tags is an optional list containing tuples of 'key=value' ########################### ALLOWLIST EXAMPLE ########################### Allowlist: Accounts: @@ -15,7 +16,10 @@ Allowlist: Regions: - "*" Resources: - - "test" # Will ignore every resource containing the string "test" in every account and region + - "test" # Will ignore every resource containing the string "test" and the tags 'test=test' and 'project=test' in account 123456789012 and every region + Tags: + - "test=test" # Will ignore every resource containing the string "test" and the tags 'test=test' and 'project=test' in account 123456789012 and every region + - "project=test" "*": Checks: @@ -27,6 +31,14 @@ Allowlist: - "ci-logs" # Will ignore bucket "ci-logs" AND ALSO bucket "ci-logs-replica" in specified check and regions - "logs" # Will ignore EVERY BUCKET containing the string "logs" in specified check and regions - "[[:alnum:]]+-logs" # Will ignore all buckets containing the terms ci-logs, qa-logs, etc. in specified check and regions + "*": + Regions: + - "*" + Resources: + - "*" + Tags: + - "environment=dev" # Will ignore every resource containing the tag 'environment=dev' in every account and region + # EXAMPLE: CONTROL TOWER (to migrate) # When using Control Tower, guardrails prevent access to certain protected resources. The allowlist diff --git a/prowler/lib/outputs/models.py b/prowler/lib/outputs/models.py index 47940c5e..cfec510d 100644 --- a/prowler/lib/outputs/models.py +++ b/prowler/lib/outputs/models.py @@ -162,7 +162,7 @@ def unroll_list(listed_items: list): def unroll_tags(tags: list): unrolled_items = "" separator = "|" - if tags: + if tags and tags != [{}] and tags != [None]: for item in tags: # Check if there are tags in list if type(item) == dict: diff --git a/prowler/lib/outputs/outputs.py b/prowler/lib/outputs/outputs.py index 8babeb2b..4e574fbc 100644 --- a/prowler/lib/outputs/outputs.py +++ b/prowler/lib/outputs/outputs.py @@ -20,6 +20,7 @@ from prowler.lib.outputs.models import ( Check_Output_JSON_ASFF, generate_provider_output_csv, generate_provider_output_json, + unroll_tags, ) from prowler.providers.aws.lib.allowlist.allowlist import is_allowlisted from prowler.providers.aws.lib.audit_info.models import AWS_Audit_Info @@ -70,6 +71,7 @@ def report(check_findings, output_options, audit_info): finding.check_metadata.CheckID, finding.region, finding.resource_id, + unroll_tags(finding.resource_tags), ): finding.status = "WARNING" # Print findings by stdout diff --git a/prowler/providers/aws/lib/allowlist/allowlist.py b/prowler/providers/aws/lib/allowlist/allowlist.py index 2f134163..5967485c 100644 --- a/prowler/providers/aws/lib/allowlist/allowlist.py +++ b/prowler/providers/aws/lib/allowlist/allowlist.py @@ -3,12 +3,20 @@ import sys import yaml from boto3.dynamodb.conditions import Attr -from schema import Schema +from schema import Optional, Schema from prowler.lib.logger import logger allowlist_schema = Schema( - {"Accounts": {str: {"Checks": {str: {"Regions": list, "Resources": list}}}}} + { + "Accounts": { + str: { + "Checks": { + str: {"Regions": list, "Resources": list, Optional("Tags"): list} + } + } + } + } ) @@ -61,14 +69,25 @@ def parse_allowlist_file(audit_info, allowlist_file): dynamodb_items.update(response["Items"]) for item in dynamodb_items: # Create allowlist for every item - allowlist["Accounts"][item["Accounts"]] = { - "Checks": { - item["Checks"]: { - "Regions": item["Regions"], - "Resources": item["Resources"], + if "Tags" in item: + allowlist["Accounts"][item["Accounts"]] = { + "Checks": { + item["Checks"]: { + "Regions": item["Regions"], + "Resources": item["Resources"], + "Tags": item["Tags"], + } + } + } + else: + allowlist["Accounts"][item["Accounts"]] = { + "Checks": { + item["Checks"]: { + "Regions": item["Regions"], + "Resources": item["Resources"], + } } } - } else: with open(allowlist_file) as f: allowlist = yaml.safe_load(f)["Allowlist"] @@ -87,18 +106,18 @@ def parse_allowlist_file(audit_info, allowlist_file): sys.exit(1) -def is_allowlisted(allowlist, audited_account, check, region, resource): +def is_allowlisted(allowlist, audited_account, check, region, resource, tags): try: if audited_account in allowlist["Accounts"]: if is_allowlisted_in_check( - allowlist, audited_account, check, region, resource + allowlist, audited_account, check, region, resource, tags ): return True # If there is a *, it affects to all accounts if "*" in allowlist["Accounts"]: audited_account = "*" if is_allowlisted_in_check( - allowlist, audited_account, check, region, resource + allowlist, audited_account, check, region, resource, tags ): return True return False @@ -109,19 +128,19 @@ def is_allowlisted(allowlist, audited_account, check, region, resource): sys.exit(1) -def is_allowlisted_in_check(allowlist, audited_account, check, region, resource): +def is_allowlisted_in_check(allowlist, audited_account, check, region, resource, tags): try: # If there is a *, it affects to all checks if "*" in allowlist["Accounts"][audited_account]["Checks"]: check = "*" if is_allowlisted_in_region( - allowlist, audited_account, check, region, resource + allowlist, audited_account, check, region, resource, tags ): return True # Check if there is the specific check if check in allowlist["Accounts"][audited_account]["Checks"]: if is_allowlisted_in_region( - allowlist, audited_account, check, region, resource + allowlist, audited_account, check, region, resource, tags ): return True return False @@ -132,30 +151,59 @@ def is_allowlisted_in_check(allowlist, audited_account, check, region, resource) sys.exit(1) -def is_allowlisted_in_region(allowlist, audited_account, check, region, resource): +def is_allowlisted_in_region(allowlist, audited_account, check, region, resource, tags): try: # If there is a *, it affects to all regions if "*" in allowlist["Accounts"][audited_account]["Checks"][check]["Regions"]: for elem in allowlist["Accounts"][audited_account]["Checks"][check][ "Resources" ]: - # Check if it is an * - if elem == "*": - elem = ".*" - if re.search(elem, resource): + if is_allowlisted_in_tags( + allowlist["Accounts"][audited_account]["Checks"][check], + elem, + resource, + tags, + ): return True # Check if there is the specific region if region in allowlist["Accounts"][audited_account]["Checks"][check]["Regions"]: for elem in allowlist["Accounts"][audited_account]["Checks"][check][ "Resources" ]: - # Check if it is an * - if elem == "*": - elem = ".*" - if re.search(elem, resource): + if is_allowlisted_in_tags( + allowlist["Accounts"][audited_account]["Checks"][check], + elem, + resource, + tags, + ): return True except Exception as error: logger.critical( f"{error.__class__.__name__} -- {error}[{error.__traceback__.tb_lineno}]" ) sys.exit(1) + + +def is_allowlisted_in_tags(check_allowlist, elem, resource, tags): + try: + # Check if it is an * + if elem == "*": + elem = ".*" + # Check if there are allowlisted tags + if "Tags" in check_allowlist: + # Check if there are resource tags + if tags: + tags_in_resource_tags = True + for tag in check_allowlist["Tags"]: + if tag not in tags: + tags_in_resource_tags = False + if tags_in_resource_tags and re.search(elem, resource): + return True + else: + if re.search(elem, resource): + return True + except Exception as error: + logger.critical( + f"{error.__class__.__name__} -- {error}[{error.__traceback__.tb_lineno}]" + ) + sys.exit(1) diff --git a/tests/providers/aws/lib/allowlist/allowlist_test.py b/tests/providers/aws/lib/allowlist/allowlist_test.py index 184c1df2..a5a259fc 100644 --- a/tests/providers/aws/lib/allowlist/allowlist_test.py +++ b/tests/providers/aws/lib/allowlist/allowlist_test.py @@ -4,6 +4,9 @@ from moto import mock_dynamodb, mock_s3 from prowler.providers.aws.lib.allowlist.allowlist import ( is_allowlisted, + is_allowlisted_in_check, + is_allowlisted_in_region, + is_allowlisted_in_tags, parse_allowlist_file, ) from prowler.providers.aws.lib.audit_info.models import AWS_Audit_Info @@ -13,7 +16,6 @@ AWS_REGION = "us-east-1" class Test_Allowlist: - # Mocked Audit Info def set_mocked_audit_info(self): audit_info = AWS_Audit_Info( @@ -56,7 +58,7 @@ class Test_Allowlist: audit_info, "s3://test-allowlist/allowlist.yaml" ) - # Test S3 allowlist + # Test DynamoDB allowlist @mock_dynamodb def test_dynamo_allowlist(self): audit_info = self.set_mocked_audit_info() @@ -101,9 +103,53 @@ class Test_Allowlist: )["Accounts"]["*"]["Checks"]["iam_user_hardware_mfa_enabled"]["Resources"] ) + @mock_dynamodb + def test_dynamo_allowlist_with_tags(self): + audit_info = self.set_mocked_audit_info() + # Create table and put item + dynamodb_resource = resource("dynamodb", region_name=AWS_REGION) + table_name = "test-allowlist" + params = { + "TableName": table_name, + "KeySchema": [ + {"AttributeName": "Accounts", "KeyType": "HASH"}, + {"AttributeName": "Checks", "KeyType": "RANGE"}, + ], + "AttributeDefinitions": [ + {"AttributeName": "Accounts", "AttributeType": "S"}, + {"AttributeName": "Checks", "AttributeType": "S"}, + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": 10, + "WriteCapacityUnits": 10, + }, + } + table = dynamodb_resource.create_table(**params) + table.put_item( + Item={ + "Accounts": "*", + "Checks": "*", + "Regions": ["*"], + "Resources": ["*"], + "Tags": ["environment=dev"], + } + ) + + assert ( + "environment=dev" + in parse_allowlist_file( + audit_info, + "arn:aws:dynamodb:" + + AWS_REGION + + ":" + + str(AWS_ACCOUNT_NUMBER) + + ":table/" + + table_name, + )["Accounts"]["*"]["Checks"]["*"]["Tags"] + ) + # Allowlist checks def test_is_allowlisted(self): - # Allowlist example allowlist = { "Accounts": { @@ -119,29 +165,33 @@ class Test_Allowlist: } assert is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler", "" ) assert is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-test" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-test", "" ) assert is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "test-prowler" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "test-prowler", "" ) assert is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-pro-test" + allowlist, + AWS_ACCOUNT_NUMBER, + "check_test", + AWS_REGION, + "prowler-pro-test", + "", ) assert not ( is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test", "" ) ) def test_is_allowlisted_wildcard(self): - # Allowlist example allowlist = { "Accounts": { @@ -157,25 +207,24 @@ class Test_Allowlist: } assert is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler", "" ) assert is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-test" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-test", "" ) assert is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "test-prowler" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "test-prowler", "" ) assert not ( is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test", "" ) ) def test_is_allowlisted_asterisk(self): - # Allowlist example allowlist = { "Accounts": { @@ -191,19 +240,161 @@ class Test_Allowlist: } assert is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler", "" ) assert is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-test" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-test", "" ) assert is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "test-prowler" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "test-prowler", "" ) assert not ( is_allowlisted( - allowlist, AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test" + allowlist, AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test", "" + ) + ) + + def test_is_allowlisted_in_region(self): + # Allowlist example + allowlist = { + "Accounts": { + AWS_ACCOUNT_NUMBER: { + "Checks": { + "check_test": { + "Regions": ["us-east-1", "eu-west-1"], + "Resources": ["*"], + } + } + } + } + } + + assert is_allowlisted_in_region( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler", "" + ) + + assert is_allowlisted_in_region( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-test", "" + ) + + assert is_allowlisted_in_region( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "test-prowler", "" + ) + + assert not ( + is_allowlisted_in_region( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test", "" + ) + ) + + def test_is_allowlisted_in_check(self): + # Allowlist example + allowlist = { + "Accounts": { + AWS_ACCOUNT_NUMBER: { + "Checks": { + "check_test": { + "Regions": ["us-east-1", "eu-west-1"], + "Resources": ["*"], + } + } + } + } + } + + assert is_allowlisted_in_check( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler", "" + ) + + assert is_allowlisted_in_check( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-test", "" + ) + + assert is_allowlisted_in_check( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "test-prowler", "" + ) + + assert not ( + is_allowlisted_in_check( + allowlist, AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test", "" + ) + ) + + def test_is_allowlisted_tags(self): + # Allowlist example + allowlist = { + "Accounts": { + "*": { + "Checks": { + "check_test": { + "Regions": ["us-east-1", "eu-west-1"], + "Resources": ["*"], + "Tags": ["environment=dev", "project=prowler"], + } + } + } + } + } + + assert not is_allowlisted( + allowlist, + AWS_ACCOUNT_NUMBER, + "check_test", + AWS_REGION, + "prowler", + "environment=dev", + ) + + assert is_allowlisted( + allowlist, + AWS_ACCOUNT_NUMBER, + "check_test", + AWS_REGION, + "prowler-test", + "environment=dev project=prowler", + ) + + assert not ( + is_allowlisted( + allowlist, + AWS_ACCOUNT_NUMBER, + "check_test", + "us-east-2", + "test", + "environment=pro", + ) + ) + + def test_is_allowlisted_in_tags(self): + # Allowlist example + check_allowlist = { + "Regions": ["us-east-1", "eu-west-1"], + "Resources": ["*"], + "Tags": ["environment=dev", "project=prowler"], + } + + assert not is_allowlisted_in_tags( + check_allowlist, + check_allowlist["Resources"][0], + "prowler", + "environment=dev", + ) + + assert is_allowlisted_in_tags( + check_allowlist, + check_allowlist["Resources"][0], + "prowler-test", + "environment=dev project=prowler", + ) + + assert not ( + is_allowlisted_in_tags( + check_allowlist, + check_allowlist["Resources"][0], + "test", + "environment=pro", ) )