From 2d832bca15af863d74c84310e019beb2af2e3ce9 Mon Sep 17 00:00:00 2001 From: Sergio Garcia <38561120+sergargar@users.noreply.github.com> Date: Thu, 3 Aug 2023 10:52:52 +0200 Subject: [PATCH] feat(gcp): Improve gcp performance (#2662) --- poetry.lock | 5 +- prowler/providers/gcp/gcp_provider.py | 4 +- prowler/providers/gcp/lib/service/service.py | 48 ++++++- .../cloudresourcemanager_service.py | 2 +- .../gcp/services/compute/compute_service.py | 134 +++++++++--------- .../gcp/services/dataproc/dataproc_service.py | 52 ++++--- .../providers/gcp/services/kms/kms_service.py | 48 +++---- pyproject.toml | 1 + 8 files changed, 165 insertions(+), 129 deletions(-) diff --git a/poetry.lock b/poetry.lock index 36ac4552..24c232e5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2383,8 +2383,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_12_6_arm64.whl", hash = "sha256:721bc4ba4525f53f6a611ec0967bdcee61b31df5a56801281027a3a6d1c2daf5"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, @@ -2876,4 +2875,4 @@ docs = ["mkdocs", "mkdocs-material"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "95a3c66d752dc8552a3e2c032545cda76f13f056ff0f0554a8664f20c8da39c4" +content-hash = "2e4af8e15db5d10b860a960d9fa3f4511182f858121e326f98aeca4a1bd75c86" diff --git a/prowler/providers/gcp/gcp_provider.py b/prowler/providers/gcp/gcp_provider.py index 7189270c..fe8ae67d 100644 --- a/prowler/providers/gcp/gcp_provider.py +++ b/prowler/providers/gcp/gcp_provider.py @@ -47,7 +47,9 @@ class GCP_Provider: if credentials_file: self.__set_gcp_creds_env_var__(credentials_file) - return auth.default() + return auth.default( + scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) except Exception as error: logger.critical( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" diff --git a/prowler/providers/gcp/lib/service/service.py b/prowler/providers/gcp/lib/service/service.py index d93f5477..43ebd45f 100644 --- a/prowler/providers/gcp/lib/service/service.py +++ b/prowler/providers/gcp/lib/service/service.py @@ -1,3 +1,10 @@ +import threading + +import google_auth_httplib2 +import httplib2 +from googleapiclient import discovery + +from prowler.lib.logger import logger from prowler.providers.gcp.gcp_provider import generate_client @@ -6,13 +13,48 @@ class GCPService: # We receive the service using __class__.__name__ or the service name in lowercase # e.g.: APIKeys --> we need a lowercase string, so service.lower() self.service = service.lower() if not service.islower() else service - + self.credentials = audit_info.credentials self.api_version = api_version - self.project_ids = audit_info.project_ids self.default_project_id = audit_info.default_project_id - self.region = region self.client = generate_client(service, api_version, audit_info) + # Only project ids that have their API enabled will be scanned + self.project_ids = self.__is_api_active__(audit_info.project_ids) def __get_client__(self): return self.client + + def __threading_call__(self, call, iterator): + threads = [] + for value in iterator: + threads.append(threading.Thread(target=call, args=(value,))) + for t in threads: + t.start() + for t in threads: + t.join() + + def __get_AuthorizedHttp_client__(self): + return google_auth_httplib2.AuthorizedHttp( + self.credentials, http=httplib2.Http() + ) + + def __is_api_active__(self, audited_project_ids): + project_ids = [] + for project_id in audited_project_ids: + try: + client = discovery.build("serviceusage", "v1") + request = client.services().get( + name=f"projects/{project_id}/services/{self.service}.googleapis.com" + ) + response = request.execute() + if response.get("state") != "DISABLED": + project_ids.append(project_id) + else: + logger.error( + f"{self.service} API has not been used in project {project_id} before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/dataproc.googleapis.com/overview?project={project_id} then retry." + ) + except Exception as error: + logger.error( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + return project_ids diff --git a/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py b/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py index a0e00bc1..55b6cf1f 100644 --- a/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py +++ b/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py @@ -43,7 +43,7 @@ class CloudResourceManager(GCPService): def __get_organizations__(self): try: response = self.client.organizations().search().execute() - for org in response["organizations"]: + for org in response.get("organizations", []): self.organizations.append( Organization(id=org["name"].split("/")[-1], name=org["displayName"]) ) diff --git a/prowler/providers/gcp/services/compute/compute_service.py b/prowler/providers/gcp/services/compute/compute_service.py index 0f797643..88e927c4 100644 --- a/prowler/providers/gcp/services/compute/compute_service.py +++ b/prowler/providers/gcp/services/compute/compute_service.py @@ -21,9 +21,9 @@ class Compute(GCPService): self.__get_regions__() self.__get_projects__() self.__get_zones__() - self.__get_instances__() + self.__threading_call__(self.__get_instances__, self.zones) self.__get_networks__() - self.__get_subnetworks__() + self.__threading_call__(self.__get_subnetworks__, self.regions) self.__get_firewalls__() def __get_regions__(self): @@ -78,60 +78,57 @@ class Compute(GCPService): f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - def __get_instances__(self): + def __get_instances__(self, zone): for project_id in self.project_ids: try: - for zone in self.zones: - request = self.client.instances().list( - project=project_id, zone=zone + request = self.client.instances().list(project=project_id, zone=zone) + while request is not None: + response = request.execute( + http=self.__get_AuthorizedHttp_client__() ) - while request is not None: - response = request.execute() - for instance in response.get("items", []): - public_ip = False - for interface in instance["networkInterfaces"]: - for config in interface.get("accessConfigs", []): - if "natIP" in config: - public_ip = True - self.instances.append( - Instance( - name=instance["name"], - id=instance["id"], - zone=zone, - public_ip=public_ip, - metadata=instance["metadata"], - shielded_enabled_vtpm=instance[ - "shieldedInstanceConfig" - ]["enableVtpm"], - shielded_enabled_integrity_monitoring=instance[ - "shieldedInstanceConfig" - ]["enableIntegrityMonitoring"], - confidential_computing=instance.get( - "confidentialInstanceConfig", {} - ).get("enableConfidentialCompute", False), - service_accounts=instance.get( - "serviceAccounts", [] - ), - ip_forward=instance.get("canIpForward", False), - disks_encryption=[ - ( - disk["deviceName"], - True - if disk.get("diskEncryptionKey", {}).get( - "sha256" - ) - else False, + for instance in response.get("items", []): + public_ip = False + for interface in instance["networkInterfaces"]: + for config in interface.get("accessConfigs", []): + if "natIP" in config: + public_ip = True + self.instances.append( + Instance( + name=instance["name"], + id=instance["id"], + zone=zone, + public_ip=public_ip, + metadata=instance["metadata"], + shielded_enabled_vtpm=instance[ + "shieldedInstanceConfig" + ]["enableVtpm"], + shielded_enabled_integrity_monitoring=instance[ + "shieldedInstanceConfig" + ]["enableIntegrityMonitoring"], + confidential_computing=instance.get( + "confidentialInstanceConfig", {} + ).get("enableConfidentialCompute", False), + service_accounts=instance.get("serviceAccounts", []), + ip_forward=instance.get("canIpForward", False), + disks_encryption=[ + ( + disk["deviceName"], + True + if disk.get("diskEncryptionKey", {}).get( + "sha256" ) - for disk in instance["disks"] - ], - project_id=project_id, - ) + else False, + ) + for disk in instance["disks"] + ], + project_id=project_id, ) - - request = self.client.instances().list_next( - previous_request=request, previous_response=response ) + + request = self.client.instances().list_next( + previous_request=request, previous_response=response + ) except Exception as error: logger.error( f"{zone} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" @@ -168,30 +165,31 @@ class Compute(GCPService): f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - def __get_subnetworks__(self): + def __get_subnetworks__(self, region): for project_id in self.project_ids: try: - for region in self.regions: - request = self.client.subnetworks().list( - project=project_id, region=region + request = self.client.subnetworks().list( + project=project_id, region=region + ) + while request is not None: + response = request.execute( + http=self.__get_AuthorizedHttp_client__() ) - while request is not None: - response = request.execute() - for subnet in response.get("items", []): - self.subnets.append( - Subnet( - name=subnet["name"], - id=subnet["id"], - project_id=project_id, - flow_logs=subnet.get("enableFlowLogs", False), - network=subnet["network"].split("/")[-1], - region=region, - ) + for subnet in response.get("items", []): + self.subnets.append( + Subnet( + name=subnet["name"], + id=subnet["id"], + project_id=project_id, + flow_logs=subnet.get("enableFlowLogs", False), + network=subnet["network"].split("/")[-1], + region=region, ) - - request = self.client.subnetworks().list_next( - previous_request=request, previous_response=response ) + + request = self.client.subnetworks().list_next( + previous_request=request, previous_response=response + ) except Exception as error: logger.error( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" diff --git a/prowler/providers/gcp/services/dataproc/dataproc_service.py b/prowler/providers/gcp/services/dataproc/dataproc_service.py index cf066309..507f870e 100644 --- a/prowler/providers/gcp/services/dataproc/dataproc_service.py +++ b/prowler/providers/gcp/services/dataproc/dataproc_service.py @@ -9,42 +9,40 @@ from prowler.providers.gcp.services.compute.compute_client import compute_client class Dataproc(GCPService): def __init__(self, audit_info): super().__init__(__class__.__name__, audit_info) + self.regions = compute_client.regions self.clusters = [] - self.__get_clusters__() + self.__threading_call__(self.__get_clusters__, self.regions) - def __get_clusters__(self): + def __get_clusters__(self, region): for project_id in self.project_ids: try: - for region in compute_client.regions: + request = ( + self.client.projects() + .regions() + .clusters() + .list(projectId=project_id, region=region) + ) + while request is not None: + response = request.execute( + http=self.__get_AuthorizedHttp_client__() + ) + + for cluster in response.get("clusters", []): + self.clusters.append( + Cluster( + name=cluster["clusterName"], + id=cluster["clusterUuid"], + encryption_config=cluster["config"]["encryptionConfig"], + project_id=project_id, + ) + ) + request = ( self.client.projects() .regions() .clusters() - .list(projectId=project_id, region=region) + .list_next(previous_request=request, previous_response=response) ) - while request is not None: - response = request.execute() - - for cluster in response.get("clusters", []): - self.clusters.append( - Cluster( - name=cluster["clusterName"], - id=cluster["clusterUuid"], - encryption_config=cluster["config"][ - "encryptionConfig" - ], - project_id=project_id, - ) - ) - - request = ( - self.client.projects() - .regions() - .clusters() - .list_next( - previous_request=request, previous_response=response - ) - ) except Exception as error: logger.error( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" diff --git a/prowler/providers/gcp/services/kms/kms_service.py b/prowler/providers/gcp/services/kms/kms_service.py index 112aa386..f5c30ba3 100644 --- a/prowler/providers/gcp/services/kms/kms_service.py +++ b/prowler/providers/gcp/services/kms/kms_service.py @@ -14,7 +14,7 @@ class KMS(GCPService): self.key_rings = [] self.crypto_keys = [] self.__get_locations__() - self.__get_key_rings__() + self.__threading_call__(self.__get_key_rings__, self.locations) self.__get_crypto_keys__() self.__get_crypto_keys_iam_policy__() @@ -44,36 +44,32 @@ class KMS(GCPService): f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - def __get_key_rings__(self): - for location in self.locations: - try: + def __get_key_rings__(self, location): + try: + request = ( + self.client.projects().locations().keyRings().list(parent=location.name) + ) + while request is not None: + response = request.execute(http=self.__get_AuthorizedHttp_client__()) + + for ring in response.get("keyRings", []): + self.key_rings.append( + KeyRing( + name=ring["name"], + project_id=location.project_id, + ) + ) + request = ( self.client.projects() .locations() .keyRings() - .list(parent=location.name) - ) - while request is not None: - response = request.execute() - - for ring in response.get("keyRings", []): - self.key_rings.append( - KeyRing( - name=ring["name"], - project_id=location.project_id, - ) - ) - - request = ( - self.client.projects() - .locations() - .keyRings() - .list_next(previous_request=request, previous_response=response) - ) - except Exception as error: - logger.error( - f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + .list_next(previous_request=request, previous_response=response) ) + except Exception as error: + logger.error( + f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) def __get_crypto_keys__(self): for ring in self.key_rings: diff --git a/pyproject.toml b/pyproject.toml index cde7b46c..390bcffe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ botocore = "1.29.165" colorama = "0.4.6" detect-secrets = "1.4.0" google-api-python-client = "2.95.0" +google-auth-httplib2 = "^0.1.0" mkdocs = {version = "1.4.3", optional = true} mkdocs-material = {version = "9.1.20", optional = true} msgraph-core = "0.2.2"