feat(allowlist): Add Allowlist feature (#1395)

config/config.py

@@ -21,6 +21,7 @@ csv_file_suffix = ".csv"
 json_file_suffix = ".json"
 json_asff_file_suffix = ".asff.json"
 config_yaml = "providers/aws/config.yaml"
+allowlist_yaml = "providers/aws/allowlist.yaml"
 
 
 def change_config_var(variable, value):

lib/check/check.py

@@ -183,6 +183,7 @@ def set_output_options(
     input_output_directory: str,
     security_hub_enabled: bool,
     output_filename: str,
+    allowlist_file: str,
 ):
     global output_options
     output_options = Output_From_Options(
@@ -191,6 +192,7 @@ def set_output_options(
         output_directory=input_output_directory,
         security_hub_enabled=security_hub_enabled,
         output_filename=output_filename,
+        allowlist_file=allowlist_file,
         # set input options here
     )
     return output_options

lib/check/models.py

@@ -15,6 +15,7 @@ class Output_From_Options:
     output_directory: str
     security_hub_enabled: bool
     output_filename: str
+    allowlist_file: str
 
 
 # Testing Pending

lib/outputs/outputs.py

@@ -24,12 +24,12 @@ from lib.outputs.models import (
     Severity,
 )
 from lib.utils.utils import file_exists, hash_sha512, open_file
+from providers.aws.lib.allowlist.allowlist import is_allowlisted
 from providers.aws.lib.security_hub.security_hub import send_to_security_hub
 
 
 def report(check_findings, output_options, audit_info):
     check_findings.sort(key=lambda x: x.region)
-
     csv_fields = []
     # check output options
     file_descriptors = {}
@@ -46,6 +46,15 @@ def report(check_findings, output_options, audit_info):
 
     if check_findings:
         for finding in check_findings:
+            # Check if finding is allowlisted
+            if is_allowlisted(
+                output_options.allowlist_file,
+                audit_info.audited_account,
+                finding.check_metadata.CheckID,
+                finding.region,
+                finding.resource_id,
+            ):
+                finding.status = "WARNING"
             # Print findings by stdout
             color = set_report_color(finding.status)
             if output_options.is_quiet and "FAIL" in finding.status:

providers/aws/allowlist.yaml

@@ -0,0 +1,37 @@
+### Account, Check and/or Region can be * to apply for all the cases
+### Resources is a list that can have either Regex or Keywords:
+###########################  ALLOWLIST EXAMPLE  ###########################
+Allowlist:
+  Accounts:
+    "123456789012":
+      Checks:
+        "iam_user_hardware_mfa_enabled":
+          Regions:
+            - "us-east-1"
+          Resources:
+            - "user-1"           # Will ignore user-1 in check iam_user_hardware_mfa_enabled
+            - "user-2"           # Will ignore user-2 in check iam_user_hardware_mfa_enabled
+        "*":
+          Regions:
+            - "*"
+          Resources:
+            - "test"             # Will ignore every resource containing the string "test" in every account and region
+
+    "098765432109":
+      Checks:
+        "s3_bucket_object_versioning":
+          Regions:
+            - "eu-west-1"
+            - "us-east-1"
+          Resources:
+            - "ci-logs"           # Will ignore bucket "ci-logs" AND ALSO bucket "ci-logs-replica" in specified check and regions
+            - "logs"              # Will ignore EVERY BUCKET containing the string "logs" in specified check and regions
+            - "[[:alnum:]]+-logs" # Will ignore all buckets containing the terms ci-logs, qa-logs, etc. in specified check and regions
+
+# EXAMPLE: CONTROL TOWER (to migrate)
+# When using Control Tower, guardrails prevent access to certain protected resources. The allowlist
+# below ensures that warnings instead of errors are reported for the affected resources.
+#extra734:aws-controltower-logs-[[:digit:]]+-[[:alpha:]\-]+
+#extra734:aws-controltower-s3-access-logs-[[:digit:]]+-[[:alpha:]\-]+
+#extra764:aws-controltower-logs-[[:digit:]]+-[[:alpha:]\-]+
+#extra764:aws-controltower-s3-access-logs-[[:digit:]]+-[[:alpha:]\-]+

providers/aws/lib/allowlist/allowlist.py

@@ -0,0 +1,105 @@
+import re
+import sys
+
+import yaml
+from boto3.dynamodb.conditions import Attr
+
+from lib.logger import logger
+
+
+def parse_allowlist_file(audit_info, allowlist_file):
+    try:
+        # Check if file is a S3 URI
+        if re.search("^s3://([^/]+)/(.*?([^/]+))$", allowlist_file):
+            bucket = allowlist_file.split("/")[2]
+            key = ("/").join(allowlist_file.split("/")[3:])
+            s3_client = audit_info.audit_session.client("s3")
+            allowlist = yaml.safe_load(
+                s3_client.get_object(Bucket=bucket, Key=key)["Body"]
+            )["Allowlist"]
+        # Check if file is a DynamoDB ARN
+        elif re.search(
+            "^arn:[aws\|aws\-cn\|aws\-us\-gov]+:dynamodb:[a-z]{2}-[north\|south\|east\|west\|central]+-[1-9]{1}:[0-9]{12}:table\/[a-zA-Z0-9._-]+$",
+            allowlist_file,
+        ):
+            allowlist = {"Accounts": {}}
+            table_region = allowlist_file.split(":")[3]
+            dynamodb_resource = audit_info.audit_session.resource(
+                "dynamodb", region_name=table_region
+            )
+            dynamo_table = dynamodb_resource.Table(allowlist_file.split("/")[1])
+            response = dynamo_table.scan(
+                FilterExpression=Attr("Accounts").is_in(
+                    [audit_info.audited_account, "*"]
+                )
+            )
+            dynamodb_items = response["Items"]
+            # Paginate through all results
+            while "LastEvaluatedKey" in dynamodb_items:
+                response = dynamo_table.scan(
+                    ExclusiveStartKey=response["LastEvaluatedKey"],
+                    FilterExpression=Attr("Accounts").is_in(
+                        [audit_info.audited_account, "*"]
+                    ),
+                )
+                dynamodb_items.update(response["Items"])
+            for item in dynamodb_items:
+                # Create allowlist for every item
+                allowlist["Accounts"][item["Accounts"]] = {
+                    "Checks": {
+                        item["Checks"]: {
+                            "Regions": item["Regions"],
+                            "Resources": item["Resources"],
+                        }
+                    }
+                }
+        else:
+            with open(allowlist_file) as f:
+                allowlist = yaml.safe_load(f)["Allowlist"]
+                print(allowlist)
+        return allowlist
+    except Exception as error:
+        logger.critical(f"{error.__class__.__name__} -- {error}")
+        sys.exit()
+
+
+def is_allowlisted(allowlist, audited_account, check, region, resource):
+    try:
+        if audited_account in allowlist["Accounts"]:
+            if is_allowlisted_in_check(allowlist, audited_account, check, region, resource):
+                return True
+        # If there is a *, it affects to all accounts
+        if "*" in allowlist["Accounts"]:
+            audited_account = "*"
+            if is_allowlisted_in_check(allowlist, audited_account, check, region, resource):
+                return True
+        return False
+    except Exception as error:
+        logger.critical(f"{error.__class__.__name__} -- {error}")
+        sys.exit()
+
+
+def is_allowlisted_in_check(allowlist, audited_account, check, region, resource):
+    # If there is a *, it affects to all checks
+    if "*" in allowlist["Accounts"][audited_account]["Checks"]:
+        check = "*"
+        if is_allowlisted_in_region(allowlist, audited_account, check, region, resource):
+            return True
+    # Check if there is the specific check
+    if check in allowlist["Accounts"][audited_account]["Checks"]:
+        if is_allowlisted_in_region(allowlist, audited_account, check, region, resource):
+            return True
+    return False
+
+
+def is_allowlisted_in_region(allowlist, audited_account, check, region, resource):
+    # If there is a *, it affects to all regions
+    if "*" in allowlist["Accounts"][audited_account]["Checks"][check]["Regions"]:
+        for elem in allowlist["Accounts"][audited_account]["Checks"][check]["Resources"]:
+            if re.search(elem, resource):
+                return True
+    # Check if there is the specific region
+    if region in allowlist["Accounts"][audited_account]["Checks"][check]["Regions"]:
+        for elem in allowlist["Accounts"][audited_account]["Checks"][check]["Resources"]:
+            if re.search(elem, resource):
+                return True

providers/aws/lib/allowlist/allowlist_test.py

@@ -0,0 +1,142 @@
+import yaml
+from boto3 import resource, session
+from moto import mock_dynamodb, mock_s3
+
+from providers.aws.lib.allowlist.allowlist import is_allowlisted, parse_allowlist_file
+from providers.aws.lib.audit_info.models import AWS_Audit_Info
+
+AWS_ACCOUNT_NUMBER = 123456789012
+AWS_REGION = "us-east-1"
+
+
+class Test_Allowlist:
+
+    # Mocked Audit Info
+    def set_mocked_audit_info(self):
+        audit_info = AWS_Audit_Info(
+            original_session=None,
+            audit_session=session.Session(
+                profile_name=None,
+                botocore_session=None,
+            ),
+            audited_account=AWS_ACCOUNT_NUMBER,
+            audited_user_id=None,
+            audited_partition="aws",
+            audited_identity_arn=None,
+            profile=None,
+            profile_region=None,
+            credentials=None,
+            assumed_role_info=None,
+            audited_regions=None,
+            organizations_metadata=None,
+        )
+        return audit_info
+
+    # Test S3 allowlist
+    @mock_s3
+    def test_s3_allowlist(self):
+        audit_info = self.set_mocked_audit_info()
+        # Create bucket and upload allowlist yaml
+        s3_resource = resource("s3", region_name=AWS_REGION)
+        s3_resource.create_bucket(Bucket="test-allowlist")
+        s3_resource.Object("test-allowlist", "allowlist.yaml").put(
+            Body=open(
+                "providers/aws/lib/allowlist/fixtures/allowlist.yaml",
+                "rb",
+            )
+        )
+
+        with open("providers/aws/lib/allowlist/fixtures/allowlist.yaml") as f:
+            assert yaml.safe_load(f)["Allowlist"] == parse_allowlist_file(
+                audit_info, "s3://test-allowlist/allowlist.yaml"
+            )
+
+    # Test S3 allowlist
+    @mock_dynamodb
+    def test_dynamo_allowlist(self):
+        audit_info = self.set_mocked_audit_info()
+        # Create table and put item
+        dynamodb_resource = resource("dynamodb", region_name=AWS_REGION)
+        table_name = "test-allowlist"
+        params = {
+            "TableName": table_name,
+            "KeySchema": [
+                {"AttributeName": "Accounts", "KeyType": "HASH"},
+                {"AttributeName": "Checks", "KeyType": "RANGE"},
+            ],
+            "AttributeDefinitions": [
+                {"AttributeName": "Accounts", "AttributeType": "S"},
+                {"AttributeName": "Checks", "AttributeType": "S"},
+            ],
+            "ProvisionedThroughput": {
+                "ReadCapacityUnits": 10,
+                "WriteCapacityUnits": 10,
+            },
+        }
+        table = dynamodb_resource.create_table(**params)
+        table.put_item(
+            Item={
+                "Accounts": "*",
+                "Checks": "iam_user_hardware_mfa_enabled",
+                "Regions": ["eu-west-1", "us-east-1"],
+                "Resources": ["keyword"],
+            }
+        )
+
+        assert (
+            "keyword"
+            in parse_allowlist_file(
+                audit_info,
+                "arn:aws:dynamodb:"
+                + AWS_REGION
+                + ":"
+                + str(AWS_ACCOUNT_NUMBER)
+                + ":table/"
+                + table_name,
+            )["Accounts"]["*"]["Checks"]["iam_user_hardware_mfa_enabled"]["Resources"]
+        )
+
+    # Allowlist checks
+    def test_is_allowlisted(self):
+
+        # Allowlist example
+        allowlist = {
+            "Accounts": {
+                "*": {
+                    "Checks": {
+                        "check_test": {
+                            "Regions": ["us-east-1", "eu-west-1"],
+                            "Resources": ["prowler", "^test"],
+                        }
+                    }
+                }
+            }
+        }
+
+        assert (
+            is_allowlisted(
+                allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler"
+            )
+            == True
+        )
+
+        assert (
+            is_allowlisted(
+                allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "prowler-test"
+            )
+            == True
+        )
+
+        assert (
+            is_allowlisted(
+                allowlist, AWS_ACCOUNT_NUMBER, "check_test", AWS_REGION, "test-prowler"
+            )
+            == True
+        )
+
+        assert (
+            is_allowlisted(
+                allowlist, AWS_ACCOUNT_NUMBER, "check_test", "us-east-2", "test"
+            )
+            == False
+        )

providers/aws/lib/allowlist/fixtures/allowlist.yaml

@@ -0,0 +1,37 @@
+### Account, Check and/or Region can be * to apply for all the cases
+### Resources is a list that can have either Regex or Keywords:
+###########################  ALLOWLIST EXAMPLE  ###########################
+Allowlist:
+  Accounts:
+    "123456789012":
+      Checks:
+        "iam_user_hardware_mfa_enabled":
+          Regions:
+            - "us-east-1"
+          Resources:
+            - "user-1"           # Will ignore user-1 in check iam_user_hardware_mfa_enabled
+            - "user-2"           # Will ignore user-2 in check iam_user_hardware_mfa_enabled
+        "*":
+          Regions:
+            - "*"
+          Resources:
+            - "test"             # Will ignore every resource containing the string "test" in every account and region
+
+    "098765432109":
+      Checks:
+        "s3_bucket_object_versioning":
+          Regions:
+            - "eu-west-1"
+            - "us-east-1"
+          Resources:
+            - "ci-logs"           # Will ignore bucket "ci-logs" AND ALSO bucket "ci-logs-replica" in specified check and regions
+            - "logs"              # Will ignore EVERY BUCKET containing the string "logs" in specified check and regions
+            - "[[:alnum:]]+-logs" # Will ignore all buckets containing the terms ci-logs, qa-logs, etc. in specified check and regions
+
+# EXAMPLE: CONTROL TOWER (to migrate)
+# When using Control Tower, guardrails prevent access to certain protected resources. The allowlist
+# below ensures that warnings instead of errors are reported for the affected resources.
+#extra734:aws-controltower-logs-[[:digit:]]+-[[:alpha:]\-]+
+#extra734:aws-controltower-s3-access-logs-[[:digit:]]+-[[:alpha:]\-]+
+#extra764:aws-controltower-logs-[[:digit:]]+-[[:alpha:]\-]+
+#extra764:aws-controltower-s3-access-logs-[[:digit:]]+-[[:alpha:]\-]+

prowler

@@ -7,6 +7,7 @@ from os import mkdir
 from os.path import isdir
 
 from config.config import (
+    allowlist_yaml,
     change_config_var,
     default_output_directory,
     output_file_timestamp,
@@ -29,6 +30,7 @@ from lib.check.checks_loader import load_checks_to_execute
 from lib.logger import logger, set_logging_config
 from lib.outputs.outputs import close_json, send_to_s3_bucket
 from providers.aws.aws_provider import provider_set_session
+from providers.aws.lib.allowlist.allowlist import parse_allowlist_file
 from providers.aws.lib.security_hub.security_hub import (
     resolve_security_hub_previous_findings,
 )
@@ -186,6 +188,12 @@ if __name__ == "__main__":
         default=None,
         help="Shodan API key used by check ec2_elastic_ip_shodan.",
     )
+    parser.add_argument(
+        "-w",
+        "--allowlist-file",
+        nargs="?",
+        help="Path for allowlist yaml file, by default is 'providers/aws/allowlist.yaml'. See default yaml for reference and format.",
+    )
     # Parse Arguments
     args = parser.parse_args()
 
@@ -299,9 +307,18 @@ if __name__ == "__main__":
             f"prowler-output-{audit_info.audited_account}-{output_file_timestamp}"
         )
 
+    # Parse content from Allowlist file and get it, if necessary, from S3
+    if args.allowlist_file:
+        allowlist_file = parse_allowlist_file(audit_info, args.allowlist_file)
+
     # Setting output options
     audit_output_options = set_output_options(
-        args.quiet, output_modes, output_directory, args.security_hub, output_filename
+        args.quiet,
+        output_modes,
+        output_directory,
+        args.security_hub,
+        output_filename,
+        allowlist_file,
     )
 
     # Execute checks