feat(securityhub): Send only FAILs but storing all in the output files (#3195)

prowler/providers/aws/lib/arguments/arguments.py

@@ -84,6 +84,11 @@ def init_parser(self):
         action="store_true",
         help="Skip updating previous findings of Prowler in Security Hub",
     )
+    aws_security_hub_subparser.add_argument(
+        "--send-sh-only-fails",
+        action="store_true",
+        help="Send only Prowler failed findings to SecurityHub",
+    )
     # AWS Quick Inventory
     aws_quick_inventory_subparser = aws_parser.add_argument_group("Quick Inventory")
     aws_quick_inventory_subparser.add_argument(

prowler/providers/aws/lib/security_hub/security_hub.py

@@ -29,7 +29,9 @@ def prepare_security_hub_findings(
             continue
 
         # Handle quiet mode
-        if output_options.is_quiet and finding.status != "FAIL":
+        if (
+            output_options.is_quiet or output_options.send_sh_only_fails
+        ) and finding.status != "FAIL":
             continue
 
         # Get the finding region

prowler/providers/common/outputs.py

@@ -69,7 +69,8 @@ class Provider_Output_Options:
         if arguments.output_directory:
             if not isdir(arguments.output_directory):
                 if arguments.output_modes:
-                    makedirs(arguments.output_directory)
+                    # exist_ok is set to True not to raise FileExistsError
+                    makedirs(arguments.output_directory, exist_ok=True)
 
 
 class Azure_Output_Options(Provider_Output_Options):
@@ -134,6 +135,7 @@ class Aws_Output_Options(Provider_Output_Options):
 
         # Security Hub Outputs
         self.security_hub_enabled = arguments.security_hub
+        self.send_sh_only_fails = arguments.send_sh_only_fails
         if arguments.security_hub:
             if not self.output_modes:
                 self.output_modes = ["json-asff"]

tests/lib/cli/parser_test.py

@@ -882,6 +882,12 @@ class Test_Parser:
         parsed = self.parser.parse(command)
         assert parsed.skip_sh_update
 
+    def test_aws_parser_send_only_fail(self):
+        argument = "--send-sh-only-fails"
+        command = [prowler_command, argument]
+        parsed = self.parser.parse(command)
+        assert parsed.send_sh_only_fails
+
     def test_aws_parser_quick_inventory_short(self):
         argument = "-i"
         command = [prowler_command, argument]

tests/providers/aws/lib/security_hub/security_hub_test.py

@@ -21,6 +21,49 @@ from tests.providers.aws.audit_info_utils import (
     set_mocked_aws_audit_info,
 )
 
+
+def get_security_hub_finding(status: str):
+    return {
+        "SchemaVersion": "2018-10-08",
+        "Id": f"prowler-iam_user_accesskey_unused-{AWS_ACCOUNT_NUMBER}-{AWS_REGION_EU_WEST_1}-ee26b0dd4",
+        "ProductArn": f"arn:aws:securityhub:{AWS_REGION_EU_WEST_1}::product/prowler/prowler",
+        "RecordState": "ACTIVE",
+        "ProductFields": {
+            "ProviderName": "Prowler",
+            "ProviderVersion": prowler_version,
+            "ProwlerResourceName": "test",
+        },
+        "GeneratorId": "prowler-iam_user_accesskey_unused",
+        "AwsAccountId": f"{AWS_ACCOUNT_NUMBER}",
+        "Types": ["Software and Configuration Checks"],
+        "FirstObservedAt": timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "UpdatedAt": timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "CreatedAt": timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "Severity": {"Label": "LOW"},
+        "Title": "Ensure Access Keys unused are disabled",
+        "Description": "test",
+        "Resources": [
+            {
+                "Type": "AwsIamAccessAnalyzer",
+                "Id": "test",
+                "Partition": "aws",
+                "Region": f"{AWS_REGION_EU_WEST_1}",
+            }
+        ],
+        "Compliance": {
+            "Status": status,
+            "RelatedRequirements": [],
+            "AssociatedStandards": [],
+        },
+        "Remediation": {
+            "Recommendation": {
+                "Text": "Run sudo yum update and cross your fingers and toes.",
+                "Url": "https://myfp.com/recommendations/dangerous_things_and_how_to_fix_them.html",
+            }
+        },
+    }
+
+
 # Mocking Security Hub Get Findings
 make_api_call = botocore.client.BaseClient._make_api_call
 
@@ -64,10 +107,13 @@ class Test_SecurityHub:
 
         return finding
 
-    def set_mocked_output_options(self, is_quiet):
+    def set_mocked_output_options(
+        self, is_quiet: bool = False, send_sh_only_fails: bool = False
+    ):
         output_options = MagicMock
         output_options.bulk_checks_metadata = {}
         output_options.is_quiet = is_quiet
+        output_options.send_sh_only_fails = send_sh_only_fails
 
         return output_options
 
@@ -98,47 +144,7 @@ class Test_SecurityHub:
             output_options,
             enabled_regions,
         ) == {
-            AWS_REGION_EU_WEST_1: [
+            AWS_REGION_EU_WEST_1: [get_security_hub_finding("PASSED")],
-                {
-                    "SchemaVersion": "2018-10-08",
-                    "Id": f"prowler-iam_user_accesskey_unused-{AWS_ACCOUNT_NUMBER}-{AWS_REGION_EU_WEST_1}-ee26b0dd4",
-                    "ProductArn": f"arn:aws:securityhub:{AWS_REGION_EU_WEST_1}::product/prowler/prowler",
-                    "RecordState": "ACTIVE",
-                    "ProductFields": {
-                        "ProviderName": "Prowler",
-                        "ProviderVersion": prowler_version,
-                        "ProwlerResourceName": "test",
-                    },
-                    "GeneratorId": "prowler-iam_user_accesskey_unused",
-                    "AwsAccountId": f"{AWS_ACCOUNT_NUMBER}",
-                    "Types": ["Software and Configuration Checks"],
-                    "FirstObservedAt": timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "UpdatedAt": timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "CreatedAt": timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "Severity": {"Label": "LOW"},
-                    "Title": "Ensure Access Keys unused are disabled",
-                    "Description": "test",
-                    "Resources": [
-                        {
-                            "Type": "AwsIamAccessAnalyzer",
-                            "Id": "test",
-                            "Partition": "aws",
-                            "Region": f"{AWS_REGION_EU_WEST_1}",
-                        }
-                    ],
-                    "Compliance": {
-                        "Status": "PASSED",
-                        "RelatedRequirements": [],
-                        "AssociatedStandards": [],
-                    },
-                    "Remediation": {
-                        "Recommendation": {
-                            "Text": "Run sudo yum update and cross your fingers and toes.",
-                            "Url": "https://myfp.com/recommendations/dangerous_things_and_how_to_fix_them.html",
-                        }
-                    },
-                }
-            ],
         }
 
     def test_prepare_security_hub_findings_quiet_INFO_finding(self):
@@ -171,7 +177,7 @@ class Test_SecurityHub:
             enabled_regions,
         ) == {AWS_REGION_EU_WEST_1: []}
 
-    def test_prepare_security_hub_findings_quiet(self):
+    def test_prepare_security_hub_findings_quiet_PASS(self):
         enabled_regions = [AWS_REGION_EU_WEST_1]
         output_options = self.set_mocked_output_options(is_quiet=True)
         findings = [self.generate_finding("PASS", AWS_REGION_EU_WEST_1)]
@@ -186,6 +192,51 @@ class Test_SecurityHub:
             enabled_regions,
         ) == {AWS_REGION_EU_WEST_1: []}
 
+    def test_prepare_security_hub_findings_quiet_FAIL(self):
+        enabled_regions = [AWS_REGION_EU_WEST_1]
+        output_options = self.set_mocked_output_options(is_quiet=True)
+        findings = [self.generate_finding("FAIL", AWS_REGION_EU_WEST_1)]
+        audit_info = set_mocked_aws_audit_info(
+            audited_regions=[AWS_REGION_EU_WEST_1, AWS_REGION_EU_WEST_2]
+        )
+
+        assert prepare_security_hub_findings(
+            findings,
+            audit_info,
+            output_options,
+            enabled_regions,
+        ) == {AWS_REGION_EU_WEST_1: [get_security_hub_finding("FAILED")]}
+
+    def test_prepare_security_hub_findings_send_sh_only_fails_PASS(self):
+        enabled_regions = [AWS_REGION_EU_WEST_1]
+        output_options = self.set_mocked_output_options(send_sh_only_fails=True)
+        findings = [self.generate_finding("PASS", AWS_REGION_EU_WEST_1)]
+        audit_info = set_mocked_aws_audit_info(
+            audited_regions=[AWS_REGION_EU_WEST_1, AWS_REGION_EU_WEST_2]
+        )
+
+        assert prepare_security_hub_findings(
+            findings,
+            audit_info,
+            output_options,
+            enabled_regions,
+        ) == {AWS_REGION_EU_WEST_1: []}
+
+    def test_prepare_security_hub_findings_send_sh_only_fails_FAIL(self):
+        enabled_regions = [AWS_REGION_EU_WEST_1]
+        output_options = self.set_mocked_output_options(send_sh_only_fails=True)
+        findings = [self.generate_finding("FAIL", AWS_REGION_EU_WEST_1)]
+        audit_info = set_mocked_aws_audit_info(
+            audited_regions=[AWS_REGION_EU_WEST_1, AWS_REGION_EU_WEST_2]
+        )
+
+        assert prepare_security_hub_findings(
+            findings,
+            audit_info,
+            output_options,
+            enabled_regions,
+        ) == {AWS_REGION_EU_WEST_1: [get_security_hub_finding("FAILED")]}
+
     def test_prepare_security_hub_findings_no_audited_regions(self):
         enabled_regions = [AWS_REGION_EU_WEST_1]
         output_options = self.set_mocked_output_options(is_quiet=False)
@@ -198,47 +249,7 @@ class Test_SecurityHub:
             output_options,
             enabled_regions,
         ) == {
-            AWS_REGION_EU_WEST_1: [
+            AWS_REGION_EU_WEST_1: [get_security_hub_finding("PASSED")],
-                {
-                    "SchemaVersion": "2018-10-08",
-                    "Id": f"prowler-iam_user_accesskey_unused-{AWS_ACCOUNT_NUMBER}-{AWS_REGION_EU_WEST_1}-ee26b0dd4",
-                    "ProductArn": f"arn:aws:securityhub:{AWS_REGION_EU_WEST_1}::product/prowler/prowler",
-                    "RecordState": "ACTIVE",
-                    "ProductFields": {
-                        "ProviderName": "Prowler",
-                        "ProviderVersion": prowler_version,
-                        "ProwlerResourceName": "test",
-                    },
-                    "GeneratorId": "prowler-iam_user_accesskey_unused",
-                    "AwsAccountId": f"{AWS_ACCOUNT_NUMBER}",
-                    "Types": ["Software and Configuration Checks"],
-                    "FirstObservedAt": timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "UpdatedAt": timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "CreatedAt": timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "Severity": {"Label": "LOW"},
-                    "Title": "Ensure Access Keys unused are disabled",
-                    "Description": "test",
-                    "Resources": [
-                        {
-                            "Type": "AwsIamAccessAnalyzer",
-                            "Id": "test",
-                            "Partition": "aws",
-                            "Region": f"{AWS_REGION_EU_WEST_1}",
-                        }
-                    ],
-                    "Compliance": {
-                        "Status": "PASSED",
-                        "RelatedRequirements": [],
-                        "AssociatedStandards": [],
-                    },
-                    "Remediation": {
-                        "Recommendation": {
-                            "Text": "Run sudo yum update and cross your fingers and toes.",
-                            "Url": "https://myfp.com/recommendations/dangerous_things_and_how_to_fix_them.html",
-                        }
-                    },
-                }
-            ],
         }
 
     @patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call)

tests/providers/common/common_outputs_test.py

@@ -96,6 +96,7 @@ class Test_Common_Output_Options:
         arguments.shodan = "test-api-key"
         arguments.only_logs = False
         arguments.unix_timestamp = False
+        arguments.send_sh_only_fails = True
 
         audit_info = self.set_mocked_aws_audit_info()
         allowlist_file = ""
@@ -105,6 +106,7 @@ class Test_Common_Output_Options:
         )
         assert isinstance(output_options, Aws_Output_Options)
         assert output_options.security_hub_enabled
+        assert output_options.send_sh_only_fails
         assert output_options.is_quiet
         assert output_options.output_modes == ["html", "csv", "json", "json-asff"]
         assert output_options.output_directory == arguments.output_directory
@@ -160,6 +162,7 @@ class Test_Common_Output_Options:
         arguments.shodan = "test-api-key"
         arguments.only_logs = False
         arguments.unix_timestamp = False
+        arguments.send_sh_only_fails = True
 
         # Mock AWS Audit Info
         audit_info = self.set_mocked_aws_audit_info()
@@ -171,6 +174,7 @@ class Test_Common_Output_Options:
         )
         assert isinstance(output_options, Aws_Output_Options)
         assert output_options.security_hub_enabled
+        assert output_options.send_sh_only_fails
         assert output_options.is_quiet
         assert output_options.output_modes == ["html", "csv", "json", "json-asff"]
         assert output_options.output_directory == arguments.output_directory