From fbb39a364e37d4c52be1fa3d66e821d1466f6f03 Mon Sep 17 00:00:00 2001 From: Sergio Garcia <38561120+sergargar@users.noreply.github.com> Date: Wed, 7 Dec 2022 19:02:05 +0100 Subject: [PATCH] feat(quick_inventory): add quick inventory (#1533) Co-authored-by: sergargar --- lib/check/check.py | 2 +- providers/aws/lib/quick_inventory/__init__.py | 0 .../lib/quick_inventory/quick_inventory.py | 216 ++++++++++++++++++ prowler | 108 +++++---- 4 files changed, 276 insertions(+), 50 deletions(-) create mode 100644 providers/aws/lib/quick_inventory/__init__.py create mode 100644 providers/aws/lib/quick_inventory/quick_inventory.py diff --git a/lib/check/check.py b/lib/check/check.py index 7ac29b26..43819606 100644 --- a/lib/check/check.py +++ b/lib/check/check.py @@ -340,5 +340,5 @@ def execute_checks( logger.error( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - bar.title = f"-> {Fore.GREEN}Scan is completed!{Style.RESET_ALL}" + bar.title = f"-> {Fore.GREEN}Scan completed!{Style.RESET_ALL}" return all_findings diff --git a/providers/aws/lib/quick_inventory/__init__.py b/providers/aws/lib/quick_inventory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/aws/lib/quick_inventory/quick_inventory.py b/providers/aws/lib/quick_inventory/quick_inventory.py new file mode 100644 index 00000000..abfbd0c7 --- /dev/null +++ b/providers/aws/lib/quick_inventory/quick_inventory.py @@ -0,0 +1,216 @@ +import csv +import json + +from alive_progress import alive_bar +from colorama import Fore, Style +from tabulate import tabulate + +from config.config import ( + csv_file_suffix, + json_file_suffix, + orange_color, + output_file_timestamp, +) +from lib.logger import logger +from providers.aws.lib.audit_info.models import AWS_Audit_Info + + +def quick_inventory(audit_info: AWS_Audit_Info, output_directory: str): + print( + f"-=- Running Quick Inventory for AWS Account {Fore.YELLOW}{audit_info.audited_account}{Style.RESET_ALL} -=-\n" + ) + resources = [] + # If not inputed regions, check all of them + if not audit_info.audited_regions: + # EC2 client for describing all regions + ec2_client = audit_info.audit_session.client("ec2", region_name="us-east-1") + # Get all the available regions + audit_info.audited_regions = [ + region["RegionName"] for region in ec2_client.describe_regions()["Regions"] + ] + + with alive_bar( + total=len(audit_info.audited_regions), + ctrl_c=False, + bar="blocks", + spinner="classic", + stats=False, + enrich_print=False, + ) as bar: + for region in sorted(audit_info.audited_regions): + bar.title = f"-> Scanning {orange_color}{region}{Style.RESET_ALL} region" + resources_in_region = [] + try: + # If us-east-1 get IAM resources + iam_client = audit_info.audit_session.client("iam") + if region == "us-east-1": + + get_roles_paginator = iam_client.get_paginator("list_roles") + for page in get_roles_paginator.paginate(): + for role in page["Roles"]: + # Avoid aws-service-role roles + if "aws-service-role" not in role["Arn"]: + resources_in_region.append(role["Arn"]) + + get_users_paginator = iam_client.get_paginator("list_users") + for page in get_users_paginator.paginate(): + for user in page["Users"]: + resources_in_region.append(user["Arn"]) + + get_groups_paginator = iam_client.get_paginator("list_groups") + for page in get_groups_paginator.paginate(): + for group in page["Groups"]: + resources_in_region.append(group["Arn"]) + + get_policies_paginator = iam_client.get_paginator("list_policies") + for page in get_policies_paginator.paginate(Scope="Local"): + for policy in page["Policies"]: + resources_in_region.append(policy["Arn"]) + + for saml_provider in iam_client.list_saml_providers()[ + "SAMLProviderList" + ]: + resources_in_region.append(saml_provider["Arn"]) + + client = audit_info.audit_session.client( + "resourcegroupstaggingapi", region_name=region + ) + # Get all the resources + resources_count = 0 + get_resources_paginator = client.get_paginator("get_resources") + for page in get_resources_paginator.paginate(): + resources_count += len(page["ResourceTagMappingList"]) + for resource in page["ResourceTagMappingList"]: + resources_in_region.append(resource["ResourceARN"]) + bar() + print( + f"Found {Fore.GREEN}{len(resources_in_region)}{Style.RESET_ALL} resources in region {Fore.YELLOW}{region}{Style.RESET_ALL}" + ) + print("\n") + + except Exception as error: + logger.error( + f"{region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + bar() + + resources.extend(resources_in_region) + bar.title = f"-> {Fore.GREEN}Quick Inventory completed!{Style.RESET_ALL}" + + inventory_table = create_inventory_table(resources) + + print( + f"\nQuick Inventory of AWS Account {Fore.YELLOW}{audit_info.audited_account}{Style.RESET_ALL}:" + ) + + print(tabulate(inventory_table, headers="keys", tablefmt="rounded_grid")) + + print(f"\nTotal resources found: {Fore.GREEN}{len(resources)}{Style.RESET_ALL}") + + create_output(resources, audit_info, output_directory) + + +def create_inventory_table(resources: list) -> dict: + + services = {} + # { "S3": + # 123, + # "IAM": + # 239, + # } + resources_type = {} + # { "S3": + # "Buckets": 13, + # "IAM": + # "Roles": 143, + # "Users": 22, + # } + for resource in sorted(resources): + service = resource.split(":")[2] + if service not in services: + services[service] = 0 + services[service] += 1 + + if service == "s3": + resource_type = "bucket" + else: + resource_type = resource.split(":")[5].split("/")[0] + if service not in resources_type: + resources_type[service] = {} + if resource_type not in resources_type[service]: + resources_type[service][resource_type] = 0 + resources_type[service][resource_type] += 1 + + # Add results to inventory table + inventory_table = { + "Service": [], + "Total": [], + "Count per resource types": [], + } + for service in services: + summary = "" + inventory_table["Service"].append(f"{service}") + inventory_table["Total"].append( + f"{Fore.GREEN}{services[service]}{Style.RESET_ALL}" + ) + for resource_type in resources_type[service]: + summary += f"{resource_type} {Fore.GREEN}{resources_type[service][resource_type]}{Style.RESET_ALL}\n" + inventory_table["Count per resource types"].append(summary) + + return inventory_table + + +def create_output(resources: list, audit_info: AWS_Audit_Info, output_directory: str): + + json_output = [] + output_file = f"{output_directory}/prowler-inventory-{audit_info.audited_account}-{output_file_timestamp}" + + for item in sorted(resources): + resource = {} + resource["AWS_AccountID"] = audit_info.audited_account + resource["AWS_Region"] = item.split(":")[3] + resource["AWS_Partition"] = item.split(":")[1] + resource["AWS_Service"] = item.split(":")[2] + resource["AWS_ResourceType"] = item.split(":")[5].split("/")[0] + resource["AWS_ResourceID"] = "" + if len(item.split("/")) > 1: + resource["AWS_ResourceID"] = item.split("/")[-1] + elif len(item.split(":")) > 6: + resource["AWS_ResourceID"] = item.split(":")[-1] + resource["AWS_ResourceARN"] = item + # Cover S3 case + if resource["AWS_Service"] == "s3": + resource["AWS_ResourceType"] = "bucket" + resource["AWS_ResourceID"] = item.split(":")[-1] + # Cover WAFv2 case + if resource["AWS_Service"] == "wafv2": + resource["AWS_ResourceType"] = "/".join(item.split(":")[-1].split("/")[:-2]) + resource["AWS_ResourceID"] = "/".join(item.split(":")[-1].split("/")[2:]) + # Cover Config case + if resource["AWS_Service"] == "config": + resource["AWS_ResourceID"] = "/".join(item.split(":")[-1].split("/")[1:]) + json_output.append(resource) + + # Serializing json + json_object = json.dumps(json_output, indent=4) + + # Writing to sample.json + with open(output_file + json_file_suffix, "w") as outfile: + outfile.write(json_object) + + csv_file = open(output_file + csv_file_suffix, "w", newline="") + csv_writer = csv.writer(csv_file) + + count = 0 + for data in json_output: + if count == 0: + header = data.keys() + csv_writer.writerow(header) + count += 1 + csv_writer.writerow(data.values()) + + csv_file.close() + + print("\nMore details in files:") + print(f" - CSV: {Fore.GREEN}{output_file+csv_file_suffix}{Style.RESET_ALL}") + print(f" - JSON: {Fore.GREEN}{output_file+json_file_suffix}{Style.RESET_ALL}") diff --git a/prowler b/prowler index e73d6dea..50e04f08 100755 --- a/prowler +++ b/prowler @@ -39,6 +39,7 @@ from lib.outputs.outputs import ( ) from providers.aws.aws_provider import aws_provider_set_session from providers.aws.lib.allowlist.allowlist import parse_allowlist_file +from providers.aws.lib.quick_inventory.quick_inventory import quick_inventory from providers.aws.lib.security_hub.security_hub import ( resolve_security_hub_previous_findings, ) @@ -204,6 +205,12 @@ if __name__ == "__main__": action="store_true", help="Send check output to AWS Security Hub", ) + parser.add_argument( + "-i", + "--quick-inventory", + action="store_true", + help="Run Prowler Quick Inventory. The inventory will be stored in an output csv by default", + ) bucket = parser.add_mutually_exclusive_group() bucket.add_argument( "-B", @@ -452,58 +459,61 @@ if __name__ == "__main__": args.verbose, ) - # Execute checks - findings = [] - if len(checks_to_execute): - findings = execute_checks( - checks_to_execute, provider, audit_info, audit_output_options - ) + if args.quick_inventory and provider == "aws": + quick_inventory(audit_info, output_directory) else: - logger.error( - "There are no checks to execute. Please, check your input arguments" - ) + # Execute checks + findings = [] + if len(checks_to_execute): + findings = execute_checks( + checks_to_execute, provider, audit_info, audit_output_options + ) + else: + logger.error( + "There are no checks to execute. Please, check your input arguments" + ) - if output_modes: - for mode in output_modes: - # Close json file if exists - if mode == "json" or mode == "json-asff": - close_json(output_filename, output_directory, mode) - if mode == "html": - add_html_footer(output_filename, output_directory) - # Send output to S3 if needed (-B / -D) - if args.output_bucket or args.output_bucket_no_assume: - output_bucket = args.output_bucket - bucket_session = audit_info.audit_session - # Check if -D was input - if args.output_bucket_no_assume: - output_bucket = args.output_bucket_no_assume - bucket_session = audit_info.original_session - send_to_s3_bucket( - output_filename, - output_directory, - mode, - output_bucket, - bucket_session, - ) + if output_modes: + for mode in output_modes: + # Close json file if exists + if mode == "json" or mode == "json-asff": + close_json(output_filename, output_directory, mode) + if mode == "html": + add_html_footer(output_filename, output_directory) + # Send output to S3 if needed (-B / -D) + if args.output_bucket or args.output_bucket_no_assume: + output_bucket = args.output_bucket + bucket_session = audit_info.audit_session + # Check if -D was input + if args.output_bucket_no_assume: + output_bucket = args.output_bucket_no_assume + bucket_session = audit_info.original_session + send_to_s3_bucket( + output_filename, + output_directory, + mode, + output_bucket, + bucket_session, + ) - # Resolve previous fails of Security Hub - if args.security_hub: - resolve_security_hub_previous_findings(output_directory, audit_info) + # Resolve previous fails of Security Hub + if args.security_hub: + resolve_security_hub_previous_findings(output_directory, audit_info) - # Display summary table - display_summary_table( - findings, - audit_info, - audit_output_options, - provider, - ) - - if compliance_framework and findings: - # Display compliance table - display_compliance_table( + # Display summary table + display_summary_table( findings, - bulk_checks_metadata, - compliance_framework, - audit_output_options.output_filename, - audit_output_options.output_directory, + audit_info, + audit_output_options, + provider, ) + + if compliance_framework and findings: + # Display compliance table + display_compliance_table( + findings, + bulk_checks_metadata, + compliance_framework, + audit_output_options.output_filename, + audit_output_options.output_directory, + )