From 611bd909ef24f8ad583c6adc167e028edbde8e0b Mon Sep 17 00:00:00 2001 From: Sergio Garcia <38561120+sergargar@users.noreply.github.com> Date: Fri, 8 Jul 2022 09:37:32 +0200 Subject: [PATCH] feat(json-asff): add json-asff ouput (#1252) * feat(json): add json output * feat(pydantic): add pydantic model to json output * feat(json-asff): add json-asff ouput * Update config/config.py Co-authored-by: Pepe Fagoaga * Update models.py * fix(comments): Resolve comments. Co-authored-by: sergargar Co-authored-by: Pepe Fagoaga --- config/config.py | 6 +- lib/check/fixtures/metadata.json | 6 +- lib/check/models.py | 29 ++++- lib/outputs/models.py | 120 ++++++++++++------ lib/outputs/outputs.py | 81 +++++++++++- .../ec2_ebs_snapshots_encrypted.metadata.json | 6 +- ..._disable_30_days_credentials.metadata.json | 6 +- ..._disable_90_days_credentials.metadata.json | 6 +- prowler | 8 +- 9 files changed, 205 insertions(+), 63 deletions(-) diff --git a/config/config.py b/config/config.py index c3701897..d8aa2988 100644 --- a/config/config.py +++ b/config/config.py @@ -12,5 +12,7 @@ aws_services_json_file = "providers/aws/aws_regions_by_service.json" default_output_directory = getcwd() + "/output" -csv_file_suffix = timestamp.strftime("%Y%m%d%H%M%S") + ".csv" -json_file_suffix = timestamp.strftime("%Y%m%d%H%M%S") + ".json" +output_file_timestamp = timestamp.strftime("%Y%m%d%H%M%S") +csv_file_suffix = f"{output_file_timestamp}.csv" +json_file_suffix = f"{output_file_timestamp}.json" +json_asff_file_suffix = f"{output_file_timestamp}.asff.json" diff --git a/lib/check/fixtures/metadata.json b/lib/check/fixtures/metadata.json index 5b821948..ecf314b4 100644 --- a/lib/check/fixtures/metadata.json +++ b/lib/check/fixtures/metadata.json @@ -35,10 +35,10 @@ "RelatedUrl": "https://serviceofficialsiteorpageforthissubject", "Remediation": { "Code": { + "CLI": "cli command or URL to the cli command location.", "NativeIaC": "code or URL to the code location.", - "Terraform": "code or URL to the code location.", - "cli": "cli command or URL to the cli command location.", - "other": "cli command or URL to the cli command location." + "Other": "cli command or URL to the cli command location.", + "Terraform": "code or URL to the code location." }, "Recommendation": { "Text": "Run sudo yum update and cross your fingers and toes.", diff --git a/lib/check/models.py b/lib/check/models.py index f2458b58..82a67756 100644 --- a/lib/check/models.py +++ b/lib/check/models.py @@ -26,7 +26,30 @@ def load_check_metadata(metadata_file: str) -> dict: return check_metadata -# Check all values +class ComplianceItem(BaseModel): + Control: List[str] + Framework: str + Group: List[str] + Version: str + + +class Code(BaseModel): + NativeIaC: str + Terraform: str + CLI: str + Other: str + + +class Recommendation(BaseModel): + Text: str + Url: str + + +class Remediation(BaseModel): + Code: Code + Recommendation: Recommendation + + class Check_Metadata_Model(BaseModel): Provider: str CheckID: str @@ -42,13 +65,13 @@ class Check_Metadata_Model(BaseModel): Description: str Risk: str RelatedUrl: str - Remediation: dict + Remediation: Remediation Categories: List[str] Tags: dict DependsOn: List[str] RelatedTo: List[str] Notes: str - Compliance: List + Compliance: List[ComplianceItem] class Check(ABC): diff --git a/lib/outputs/models.py b/lib/outputs/models.py index 42747285..6a31c53a 100644 --- a/lib/outputs/models.py +++ b/lib/outputs/models.py @@ -1,10 +1,10 @@ from dataclasses import asdict, dataclass -from typing import Optional +from typing import List, Optional from pydantic import BaseModel from config.config import timestamp -from lib.check.models import Check_Report +from lib.check.models import Check_Report, ComplianceItem, Remediation from providers.aws.models import AWS_Organizations_Info @@ -17,36 +17,78 @@ class Compliance_Framework: class Check_Output_JSON(BaseModel): - AssessmentStartTime: Optional[str] - FindingUniqueId: Optional[str] + AssessmentStartTime: str = "" + FindingUniqueId: str = "" Provider: str - Profile: Optional[str] - AccountId: Optional[str] - OrganizationsInfo: Optional[dict] - Region: Optional[str] + Profile: str = "" + AccountId: str = "" + OrganizationsInfo: Optional[AWS_Organizations_Info] + Region: str = "" CheckID: str CheckName: str CheckTitle: str CheckType: str ServiceName: str SubServiceName: str - Status: Optional[str] - StatusExtended: Optional[str] + Status: str = "" + StatusExtended: str = "" Severity: str - ResourceId: Optional[str] - ResourceArn: Optional[str] + ResourceId: str = "" + ResourceArn: str = "" ResourceType: str - ResourceDetails: Optional[str] + ResourceDetails: str = "" Tags: dict Description: str Risk: str RelatedUrl: str - Remediation: dict - Categories: list - DependsOn: list - RelatedTo: list + Remediation: Remediation + Categories: List[str] + DependsOn: List[str] + RelatedTo: List[str] Notes: str - Compliance: list + Compliance: List[ComplianceItem] + + +# JSON ASFF Output +class ProductFields(BaseModel): + ProviderName: str = "Prowler" + ProviderVersion: str + ProwlerResourceName: str + + +class Severity(BaseModel): + Label: str + + +class Resource(BaseModel): + Type: str + Id: str + Partition: str + Region: str + + +class Compliance(BaseModel): + Status: str + RelatedRequirements: List[str] + + +class Check_Output_JSON_ASFF(BaseModel): + SchemaVersion: str = "2018-10-08" + Id: str = "" + ProductArn: str = "" + RecordState: str = "ACTIVE" + ProductFields: ProductFields = None + GeneratorId: str = "" + AwsAccountId: str = "" + Types: List[str] = None + FirstObservedAt: str = "" + UpdatedAt: str = "" + CreatedAt: str = "" + Severity: Severity = None + Title: str = "" + Description: str = "" + Resources: List[Resource] = None + Compliance: Compliance = None @dataclass @@ -133,27 +175,27 @@ class Check_Output_CSV: self.description = report.check_metadata.Description self.risk = report.check_metadata.Risk self.related_url = report.check_metadata.RelatedUrl - self.remediation_recommendation_text = report.check_metadata.Remediation[ - "Recommendation" - ]["Text"] - self.remediation_recommendation_url = report.check_metadata.Remediation[ - "Recommendation" - ]["Url"] + self.remediation_recommendation_text = ( + report.check_metadata.Remediation.Recommendation.Text + ) + self.remediation_recommendation_url = ( + report.check_metadata.Remediation.Recommendation.Url + ) self.remediation_recommendation_code_nativeiac = ( - report.check_metadata.Remediation["Code"]["NativeIaC"] + report.check_metadata.Remediation.Code.NativeIaC ) self.remediation_recommendation_code_terraform = ( - report.check_metadata.Remediation["Code"]["Terraform"] + report.check_metadata.Remediation.Code.Terraform + ) + self.remediation_recommendation_code_cli = ( + report.check_metadata.Remediation.Code.cli + ) + self.remediation_recommendation_code_cli = ( + report.check_metadata.Remediation.Code.cli + ) + self.remediation_recommendation_code_other = ( + report.check_metadata.Remediation.Code.other ) - self.remediation_recommendation_code_cli = report.check_metadata.Remediation[ - "Code" - ]["cli"] - self.remediation_recommendation_code_cli = report.check_metadata.Remediation[ - "Code" - ]["cli"] - self.remediation_recommendation_code_other = report.check_metadata.Remediation[ - "Code" - ]["other"] self.categories = self.__unroll_list__(report.check_metadata.Categories) self.depends_on = self.__unroll_list__(report.check_metadata.DependsOn) self.related_to = self.__unroll_list__(report.check_metadata.RelatedTo) @@ -188,10 +230,10 @@ class Check_Output_CSV: # fill list of dataclasses for item in compliance: compliance_framework = Compliance_Framework( - Framework=item["Framework"], - Version=item["Version"], - Group=item["Group"], - Control=item["Control"], + Framework=item.Framework, + Version=item.Version, + Group=item.Group, + Control=item.Control, ) compliance_frameworks.append(compliance_framework) # iterate over list of dataclasses to output info diff --git a/lib/outputs/outputs.py b/lib/outputs/outputs.py index 3cf7a784..af928960 100644 --- a/lib/outputs/outputs.py +++ b/lib/outputs/outputs.py @@ -4,8 +4,22 @@ from csv import DictWriter from colorama import Fore, Style -from config.config import csv_file_suffix, json_file_suffix, timestamp -from lib.outputs.models import Check_Output_CSV, Check_Output_JSON +from config.config import ( + csv_file_suffix, + json_asff_file_suffix, + json_file_suffix, + prowler_version, + timestamp, +) +from lib.outputs.models import ( + Check_Output_CSV, + Check_Output_JSON, + Check_Output_JSON_ASFF, + Compliance, + ProductFields, + Resource, + Severity, +) from lib.utils.utils import file_exists, open_file @@ -60,6 +74,15 @@ def report(check_findings, output_options, audit_info): json.dump(finding_output.dict(), file_descriptors["json"], indent=4) file_descriptors["json"].write(",") + if "json-asff" in file_descriptors: + finding_output = Check_Output_JSON_ASFF() + fill_json_asff(finding_output, audit_info, finding) + + json.dump( + finding_output.dict(), file_descriptors["json-asff"], indent=4 + ) + file_descriptors["json-asff"].write(",") + if file_descriptors: # Close all file descriptors for file_descriptor in file_descriptors: @@ -107,6 +130,22 @@ def fill_file_descriptors(output_modes, audited_account, output_directory, csv_f file_descriptors.update({output_mode: file_descriptor}) + if output_mode == "json-asff": + filename = f"{output_directory}/prowler-output-{audited_account}-{json_asff_file_suffix}" + if file_exists(filename): + file_descriptor = open_file( + filename, + "a", + ) + else: + file_descriptor = open_file( + filename, + "a", + ) + file_descriptor.write("[") + + file_descriptors.update({output_mode: file_descriptor}) + return file_descriptors @@ -149,12 +188,46 @@ def fill_json(finding_output, audit_info, finding): return finding_output -def close_json(output_directory, audited_account): - filename = f"{output_directory}/prowler-output-{audited_account}-{json_file_suffix}" +def fill_json_asff(finding_output, audit_info, finding): + finding_output.Id = f"prowler-{finding.check_metadata.CheckID}-{audit_info.audited_account}-{finding.region}-{str(hash(finding.resource_id))}" + finding_output.ProductArn = f"arn:{audit_info.audited_partition}:securityhub:{finding.region}::product/prowler/prowler" + finding_output.ProductFields = ProductFields( + ProviderVersion=prowler_version, ProwlerResourceName=finding.resource_id + ) + finding_output.GeneratorId = "prowler-" + finding.check_metadata.CheckID + finding_output.AwsAccountId = audit_info.audited_account + finding_output.Types = finding.check_metadata.CheckType + finding_output.FirstObservedAt = ( + finding_output.UpdatedAt + ) = finding_output.CreatedAt = timestamp.isoformat() + finding_output.Severity = Severity(Label=finding.check_metadata.Severity) + finding_output.Title = finding.check_metadata.CheckTitle + finding_output.Description = finding.check_metadata.Description + finding_output.Resources = [ + Resource( + Id=finding.resource_id, + Type=finding.check_metadata.ResourceType, + Partition=audit_info.audited_partition, + Region=finding.region, + ) + ] + finding_output.Compliance = Compliance( + Status=finding.status, RelatedRequirements=[finding.check_metadata.CheckType] + ) + + return finding_output + + +def close_json(output_directory, audited_account, mode): + suffix = json_file_suffix + if mode == "json-asff": + suffix = json_asff_file_suffix + filename = f"{output_directory}/prowler-output-{audited_account}-{suffix}" file_descriptor = open_file( filename, "a", ) + # Replace last comma for square bracket file_descriptor.seek(file_descriptor.tell() - 1, os.SEEK_SET) file_descriptor.truncate() file_descriptor.write("]") diff --git a/providers/aws/services/ec2/ec2_ebs_snapshots_encrypted/ec2_ebs_snapshots_encrypted.metadata.json b/providers/aws/services/ec2/ec2_ebs_snapshots_encrypted/ec2_ebs_snapshots_encrypted.metadata.json index ef7b5a94..c8e5e5b5 100644 --- a/providers/aws/services/ec2/ec2_ebs_snapshots_encrypted/ec2_ebs_snapshots_encrypted.metadata.json +++ b/providers/aws/services/ec2/ec2_ebs_snapshots_encrypted/ec2_ebs_snapshots_encrypted.metadata.json @@ -46,10 +46,10 @@ "RelatedUrl": "https://serviceofficialsiteorpageforthissubject", "Remediation": { "Code": { + "CLI": "cli command or URL to the cli command location.", "NativeIaC": "code or URL to the code location.", - "Terraform": "code or URL to the code location.", - "cli": "cli command or URL to the cli command location.", - "other": "cli command or URL to the cli command location." + "Other": "cli command or URL to the cli command location.", + "Terraform": "code or URL to the code location." }, "Recommendation": { "Text": "Run sudo yum update and cross your fingers and toes.", diff --git a/providers/aws/services/iam/iam_disable_30_days_credentials/iam_disable_30_days_credentials.metadata.json b/providers/aws/services/iam/iam_disable_30_days_credentials/iam_disable_30_days_credentials.metadata.json index 05e1d6a0..bb970fdd 100644 --- a/providers/aws/services/iam/iam_disable_30_days_credentials/iam_disable_30_days_credentials.metadata.json +++ b/providers/aws/services/iam/iam_disable_30_days_credentials/iam_disable_30_days_credentials.metadata.json @@ -35,10 +35,10 @@ "RelatedUrl": "https://serviceofficialsiteorpageforthissubject", "Remediation": { "Code": { + "CLI": "cli command or URL to the cli command location.", "NativeIaC": "code or URL to the code location.", - "Terraform": "code or URL to the code location.", - "cli": "cli command or URL to the cli command location.", - "other": "cli command or URL to the cli command location." + "Other": "cli command or URL to the cli command location.", + "Terraform": "code or URL to the code location." }, "Recommendation": { "Text": "Run sudo yum update and cross your fingers and toes.", diff --git a/providers/aws/services/iam/iam_disable_90_days_credentials/iam_disable_90_days_credentials.metadata.json b/providers/aws/services/iam/iam_disable_90_days_credentials/iam_disable_90_days_credentials.metadata.json index 3fdaddd2..75388b1d 100644 --- a/providers/aws/services/iam/iam_disable_90_days_credentials/iam_disable_90_days_credentials.metadata.json +++ b/providers/aws/services/iam/iam_disable_90_days_credentials/iam_disable_90_days_credentials.metadata.json @@ -35,10 +35,10 @@ "RelatedUrl": "https://serviceofficialsiteorpageforthissubject", "Remediation": { "Code": { + "CLI": "cli command or URL to the cli command location.", "NativeIaC": "code or URL to the code location.", - "Terraform": "code or URL to the code location.", - "cli": "cli command or URL to the cli command location.", - "other": "cli command or URL to the cli command location." + "Other": "cli command or URL to the cli command location.", + "Terraform": "code or URL to the code location." }, "Recommendation": { "Text": "Run sudo yum update and cross your fingers and toes.", diff --git a/prowler b/prowler index cfdda49c..a7731c20 100755 --- a/prowler +++ b/prowler @@ -125,7 +125,7 @@ if __name__ == "__main__": "--output-modes", nargs="+", help="Output mode, by default csv", - choices=["csv", "json"], + choices=["csv", "json", "json-asff"], ) parser.add_argument( "-o", @@ -272,5 +272,7 @@ if __name__ == "__main__": ) # Close json file if exists - if "json" in output_modes: - close_json(output_directory, audit_info.audited_account) + if output_modes: + for mode in output_modes: + if mode == "json" or mode == "json-asff": + close_json(output_directory, audit_info.audited_account, mode)