feat(boto3-config): Use standard retrier (#1868)

Co-authored-by: Sergio Garcia <38561120+sergargar@users.noreply.github.com>
This commit is contained in:
Pepe Fagoaga
2023-02-09 09:58:47 +01:00
committed by GitHub
parent 9df7def14e
commit de281535b1
63 changed files with 105 additions and 1 deletions

View File

@@ -379,6 +379,16 @@ Detailed documentation at https://docs.prowler.cloud
help="Scan only resources with specific AWS Resource ARNs, e.g., arn:aws:iam::012345678910:user/test arn:aws:ec2:us-east-1:123456789012:vpc/vpc-12345678",
)
# Boto3 Config
boto3_config_subparser = aws_parser.add_argument_group("Boto3 Config")
boto3_config_subparser.add_argument(
"--aws-retries-max-attempts",
nargs="?",
default=None,
type=int,
help="Set the maximum attemps for the Boto3 standard retrier config (Default: 3)",
)
def __init_azure_parser__(self):
"""Init the Azure Provider CLI parser"""
azure_parser = self.subparsers.add_parser(

View File

@@ -130,7 +130,7 @@ def generate_regional_clients(
regions = regions[:1]
for region in regions:
regional_client = audit_info.audit_session.client(
service, region_name=region
service, region_name=region, config=audit_info.session_config
)
regional_client.region = region
regional_clients[region] = regional_client

View File

@@ -1,4 +1,5 @@
from boto3 import session
from botocore.config import Config
from prowler.providers.aws.lib.audit_info.models import AWS_Assume_Role, AWS_Audit_Info
@@ -9,6 +10,9 @@ current_audit_info = AWS_Audit_Info(
profile_name=None,
botocore_session=None,
),
# Default standard retrier config
# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html
session_config=Config(retries={"max_attempts": 3, "mode": "standard"}),
audited_account=None,
audited_user_id=None,
audited_partition=None,

View File

@@ -3,6 +3,7 @@ from datetime import datetime
from typing import Any, Optional
from boto3 import session
from botocore.config import Config
@dataclass
@@ -33,6 +34,8 @@ class AWS_Organizations_Info:
class AWS_Audit_Info:
original_session: session.Session
audit_session: session.Session
# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html
session_config: Config
audited_account: int
audited_identity_arn: str
audited_user_id: str

View File

@@ -2,6 +2,7 @@ import sys
from arnparse import arnparse
from boto3 import client, session
from botocore.config import Config
from colorama import Fore, Style
from prowler.lib.logger import logger
@@ -123,6 +124,20 @@ Caller Identity ARN: {Fore.YELLOW}[{audit_info.audited_identity_arn}]{Style.RESE
# Assumed AWS session
assumed_session = None
# Set the maximum retries for the standard retrier config
aws_retries_max_attempts = arguments.get("aws_retries_max_attempts")
if aws_retries_max_attempts:
# Create the new config
config = Config(
retries={
"max_attempts": aws_retries_max_attempts,
"mode": "standard",
}
)
# Merge the new configuration
new_boto3_config = current_audit_info.session_config.merge(config)
current_audit_info.session_config = new_boto3_config
# Setting session
current_audit_info.profile = input_profile
current_audit_info.audited_regions = input_regions

View File

@@ -834,6 +834,13 @@ class Test_Parser:
self.parser.parse(command)
assert ex.type == SystemExit
def test_aws_parser_aws_retries_max_attempts(self):
argument = "--aws-retries-max-attempts"
max_retries = "10"
command = [prowler_command, argument, max_retries]
parsed = self.parser.parse(command)
assert parsed.aws_retries_max_attempts == int(max_retries)
def test_parser_azure_auth_sp(self):
argument = "--sp-env-auth"
command = [prowler_command, "azure", argument]

View File

@@ -68,6 +68,7 @@ class Test_Outputs:
audited_account = AWS_ACCOUNT_ID
output_directory = f"{os.path.dirname(os.path.realpath(__file__))}"
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=None,
audited_account=AWS_ACCOUNT_ID,
@@ -202,6 +203,7 @@ class Test_Outputs:
# def test_fill_json(self):
# input_audit_info = AWS_Audit_Info(
session_config = (None,)
# original_session=None,
# audit_session=None,
# audited_account=AWS_ACCOUNT_ID,
@@ -246,6 +248,7 @@ class Test_Outputs:
def test_fill_json_asff(self):
input_audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=None,
audited_account=AWS_ACCOUNT_ID,
@@ -316,6 +319,7 @@ class Test_Outputs:
)
# Create mock audit_info
input_audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session,
audited_account=AWS_ACCOUNT_ID,
@@ -364,6 +368,7 @@ class Test_Outputs:
)
# Create mock audit_info
input_audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session,
audited_account=AWS_ACCOUNT_ID,
@@ -470,6 +475,7 @@ class Test_Outputs:
region_name="eu-west-1",
)
input_audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session,
audited_account=AWS_ACCOUNT_ID,

View File

@@ -40,6 +40,7 @@ class Test_AWS_Provider:
# Fulfil the input session object for Prowler
audit_info = AWS_Audit_Info(
session_config=None,
original_session=session,
audit_session=None,
audited_account=None,
@@ -98,6 +99,7 @@ class Test_AWS_Provider:
audited_regions = ["eu-west-1", "us-east-1"]
# Fulfil the input session object for Prowler
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session,
audited_account=None,
@@ -127,6 +129,7 @@ class Test_AWS_Provider:
profile_region = "us-east-1"
# Fulfil the input session object for Prowler
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session,
audited_account=None,
@@ -155,6 +158,7 @@ class Test_AWS_Provider:
audited_regions = ["cn-northwest-1", "cn-north-1"]
# Fulfil the input session object for Prowler
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session,
audited_account=None,

View File

@@ -17,6 +17,7 @@ class Test_Allowlist:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ class Test_ACM_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ class Test_APIGateway_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -43,6 +43,7 @@ class Test_ApiGatewayV2_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -14,6 +14,7 @@ class Test_AutoScaling_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -27,6 +27,7 @@ class Test_awslambda_function_invoke_api_operations_cloudtrail_logging_enabled:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -59,6 +59,7 @@ def mock_generate_regional_clients(service, audit_info):
class Test_Lambda_Service:
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -136,6 +136,7 @@ class Test_CloudFormation_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -145,6 +145,7 @@ class Test_CloudFront_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ AWS_ACCOUNT_NUMBER = 123456789012
class Test_cloudtrail_multi_region_enabled:
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ AWS_ACCOUNT_NUMBER = 123456789012
class Test_cloudtrail_s3_dataevents_read_enabled:
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ AWS_ACCOUNT_NUMBER = 123456789012
class Test_cloudtrail_s3_dataevents_write_enabled:
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -11,6 +11,7 @@ class Test_Cloudtrail_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ class Test_CloudWatch_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -55,6 +55,7 @@ def mock_generate_regional_clients(service, audit_info):
class Test_Codebuild_Service:
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ class Test_Config_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ class Test_DynamoDB_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -21,6 +21,7 @@ class Test_EC2_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -72,6 +72,7 @@ class Test_ECR_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -24,6 +24,7 @@ class Test_ECS_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -56,6 +56,7 @@ def mock_generate_regional_clients(service, audit_info):
class Test_EFS:
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -29,6 +29,7 @@ class Test_EKS_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ class Test_ELB_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ class Test_ELBv2_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -15,6 +15,7 @@ class Test_emr_cluster_publicly_accesible:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -51,6 +51,7 @@ def mock_generate_regional_clients(service, audit_info):
class Test_EMR_Service:
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -50,6 +50,7 @@ class Test_GlobalAccelerator_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -120,6 +120,7 @@ class Test_Glue_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -34,6 +34,7 @@ class Test_GuardDuty_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -15,6 +15,7 @@ class Test_IAM_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -14,6 +14,7 @@ class Test_ACM_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -100,6 +100,7 @@ class Test_OpenSearchService_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ class Test_RDS_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -60,6 +60,7 @@ class Test_Redshift_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -28,6 +28,7 @@ class Test_Route53_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -67,6 +67,7 @@ class Test_Route53_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -13,6 +13,7 @@ class Test_s3_account_level_public_access_blocks:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -14,6 +14,7 @@ class Test_s3_bucket_public_access:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -14,6 +14,7 @@ class Test_S3_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -101,6 +101,7 @@ class Test_SageMaker_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -30,6 +30,7 @@ def mock_generate_regional_clients(service, audit_info):
class Test_SecretsManager_Service:
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -27,6 +27,7 @@ class Test_shield_advanced_protection_in_associated_elastic_ips:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -14,6 +14,7 @@ class Test_shield_advanced_protection_in_classic_load_balancers:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -27,6 +27,7 @@ class Test_shield_advanced_protection_in_internet_facing_load_balancers:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -37,6 +37,7 @@ class Test_Shield_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -51,6 +51,7 @@ class Test_SNS_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -53,6 +53,7 @@ class Test_SQS_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -126,6 +126,7 @@ class Test_SSM_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -26,6 +26,7 @@ class Test_TrustedAdvisor_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -14,6 +14,7 @@ class Test_VPC_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -48,6 +48,7 @@ class Test_WAF_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -12,6 +12,7 @@ class Test_WAFv2_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -45,6 +45,7 @@ class Test_WorkSpaces_Service:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,

View File

@@ -26,6 +26,7 @@ from prowler.providers.common.audit_info import (
EXAMPLE_AMI_ID = "ami-12c6146b"
ACCOUNT_ID = 123456789012
mock_current_audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=None,
audited_account="123456789012",

View File

@@ -38,6 +38,7 @@ class Test_Common_Output_Options:
# Mocked AWS Audit Info
def set_mocked_aws_audit_info(self):
audit_info = AWS_Audit_Info(
session_config=None,
original_session=None,
audit_session=session.Session(
profile_name=None,