feat(gcp): Improve gcp performance (#2662)

poetry.lock

@@ -2383,8 +2383,7 @@ files = [
     {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"},
     {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"},
     {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"},
-    {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"},
+    {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_12_6_arm64.whl", hash = "sha256:721bc4ba4525f53f6a611ec0967bdcee61b31df5a56801281027a3a6d1c2daf5"},
-    {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"},
     {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"},
     {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"},
     {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"},
@@ -2876,4 +2875,4 @@ docs = ["mkdocs", "mkdocs-material"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "95a3c66d752dc8552a3e2c032545cda76f13f056ff0f0554a8664f20c8da39c4"
+content-hash = "2e4af8e15db5d10b860a960d9fa3f4511182f858121e326f98aeca4a1bd75c86"

prowler/providers/gcp/gcp_provider.py

@@ -47,7 +47,9 @@ class GCP_Provider:
             if credentials_file:
                 self.__set_gcp_creds_env_var__(credentials_file)
 
-            return auth.default()
+            return auth.default(
+                scopes=["https://www.googleapis.com/auth/cloud-platform"]
+            )
         except Exception as error:
             logger.critical(
                 f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"

prowler/providers/gcp/lib/service/service.py

@@ -1,3 +1,10 @@
+import threading
+
+import google_auth_httplib2
+import httplib2
+from googleapiclient import discovery
+
+from prowler.lib.logger import logger
 from prowler.providers.gcp.gcp_provider import generate_client
 
 
@@ -6,13 +13,48 @@ class GCPService:
         # We receive the service using __class__.__name__ or the service name in lowercase
         # e.g.: APIKeys --> we need a lowercase string, so service.lower()
         self.service = service.lower() if not service.islower() else service
-
+        self.credentials = audit_info.credentials
         self.api_version = api_version
-        self.project_ids = audit_info.project_ids
         self.default_project_id = audit_info.default_project_id
-
         self.region = region
         self.client = generate_client(service, api_version, audit_info)
+        # Only project ids that have their API enabled will be scanned
+        self.project_ids = self.__is_api_active__(audit_info.project_ids)
 
     def __get_client__(self):
         return self.client
+
+    def __threading_call__(self, call, iterator):
+        threads = []
+        for value in iterator:
+            threads.append(threading.Thread(target=call, args=(value,)))
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+    def __get_AuthorizedHttp_client__(self):
+        return google_auth_httplib2.AuthorizedHttp(
+            self.credentials, http=httplib2.Http()
+        )
+
+    def __is_api_active__(self, audited_project_ids):
+        project_ids = []
+        for project_id in audited_project_ids:
+            try:
+                client = discovery.build("serviceusage", "v1")
+                request = client.services().get(
+                    name=f"projects/{project_id}/services/{self.service}.googleapis.com"
+                )
+                response = request.execute()
+                if response.get("state") != "DISABLED":
+                    project_ids.append(project_id)
+                else:
+                    logger.error(
+                        f"{self.service} API has not been used in project {project_id} before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/dataproc.googleapis.com/overview?project={project_id} then retry."
+                    )
+            except Exception as error:
+                logger.error(
+                    f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
+                )
+        return project_ids

prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py

@@ -43,7 +43,7 @@ class CloudResourceManager(GCPService):
     def __get_organizations__(self):
         try:
             response = self.client.organizations().search().execute()
-            for org in response["organizations"]:
+            for org in response.get("organizations", []):
                 self.organizations.append(
                     Organization(id=org["name"].split("/")[-1], name=org["displayName"])
                 )

prowler/providers/gcp/services/compute/compute_service.py

@@ -21,9 +21,9 @@ class Compute(GCPService):
         self.__get_regions__()
         self.__get_projects__()
         self.__get_zones__()
-        self.__get_instances__()
+        self.__threading_call__(self.__get_instances__, self.zones)
         self.__get_networks__()
-        self.__get_subnetworks__()
+        self.__threading_call__(self.__get_subnetworks__, self.regions)
         self.__get_firewalls__()
 
     def __get_regions__(self):
@@ -78,60 +78,57 @@ class Compute(GCPService):
                     f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
                 )
 
-    def __get_instances__(self):
+    def __get_instances__(self, zone):
         for project_id in self.project_ids:
             try:
-                for zone in self.zones:
+                request = self.client.instances().list(project=project_id, zone=zone)
-                    request = self.client.instances().list(
+                while request is not None:
-                        project=project_id, zone=zone
+                    response = request.execute(
+                        http=self.__get_AuthorizedHttp_client__()
                     )
-                    while request is not None:
-                        response = request.execute()
 
-                        for instance in response.get("items", []):
+                    for instance in response.get("items", []):
-                            public_ip = False
+                        public_ip = False
-                            for interface in instance["networkInterfaces"]:
+                        for interface in instance["networkInterfaces"]:
-                                for config in interface.get("accessConfigs", []):
+                            for config in interface.get("accessConfigs", []):
-                                    if "natIP" in config:
+                                if "natIP" in config:
-                                        public_ip = True
+                                    public_ip = True
-                            self.instances.append(
+                        self.instances.append(
-                                Instance(
+                            Instance(
-                                    name=instance["name"],
+                                name=instance["name"],
-                                    id=instance["id"],
+                                id=instance["id"],
-                                    zone=zone,
+                                zone=zone,
-                                    public_ip=public_ip,
+                                public_ip=public_ip,
-                                    metadata=instance["metadata"],
+                                metadata=instance["metadata"],
-                                    shielded_enabled_vtpm=instance[
+                                shielded_enabled_vtpm=instance[
-                                        "shieldedInstanceConfig"
+                                    "shieldedInstanceConfig"
-                                    ]["enableVtpm"],
+                                ]["enableVtpm"],
-                                    shielded_enabled_integrity_monitoring=instance[
+                                shielded_enabled_integrity_monitoring=instance[
-                                        "shieldedInstanceConfig"
+                                    "shieldedInstanceConfig"
-                                    ]["enableIntegrityMonitoring"],
+                                ]["enableIntegrityMonitoring"],
-                                    confidential_computing=instance.get(
+                                confidential_computing=instance.get(
-                                        "confidentialInstanceConfig", {}
+                                    "confidentialInstanceConfig", {}
-                                    ).get("enableConfidentialCompute", False),
+                                ).get("enableConfidentialCompute", False),
-                                    service_accounts=instance.get(
+                                service_accounts=instance.get("serviceAccounts", []),
-                                        "serviceAccounts", []
+                                ip_forward=instance.get("canIpForward", False),
-                                    ),
+                                disks_encryption=[
-                                    ip_forward=instance.get("canIpForward", False),
+                                    (
-                                    disks_encryption=[
+                                        disk["deviceName"],
-                                        (
+                                        True
-                                            disk["deviceName"],
+                                        if disk.get("diskEncryptionKey", {}).get(
-                                            True
+                                            "sha256"
-                                            if disk.get("diskEncryptionKey", {}).get(
-                                                "sha256"
-                                            )
-                                            else False,
                                         )
-                                        for disk in instance["disks"]
+                                        else False,
-                                    ],
+                                    )
-                                    project_id=project_id,
+                                    for disk in instance["disks"]
-                                )
+                                ],
+                                project_id=project_id,
                             )
-
-                        request = self.client.instances().list_next(
-                            previous_request=request, previous_response=response
                         )
+
+                    request = self.client.instances().list_next(
+                        previous_request=request, previous_response=response
+                    )
             except Exception as error:
                 logger.error(
                     f"{zone} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
@@ -168,30 +165,31 @@ class Compute(GCPService):
                     f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
                 )
 
-    def __get_subnetworks__(self):
+    def __get_subnetworks__(self, region):
         for project_id in self.project_ids:
             try:
-                for region in self.regions:
+                request = self.client.subnetworks().list(
-                    request = self.client.subnetworks().list(
+                    project=project_id, region=region
-                        project=project_id, region=region
+                )
+                while request is not None:
+                    response = request.execute(
+                        http=self.__get_AuthorizedHttp_client__()
                     )
-                    while request is not None:
+                    for subnet in response.get("items", []):
-                        response = request.execute()
+                        self.subnets.append(
-                        for subnet in response.get("items", []):
+                            Subnet(
-                            self.subnets.append(
+                                name=subnet["name"],
-                                Subnet(
+                                id=subnet["id"],
-                                    name=subnet["name"],
+                                project_id=project_id,
-                                    id=subnet["id"],
+                                flow_logs=subnet.get("enableFlowLogs", False),
-                                    project_id=project_id,
+                                network=subnet["network"].split("/")[-1],
-                                    flow_logs=subnet.get("enableFlowLogs", False),
+                                region=region,
-                                    network=subnet["network"].split("/")[-1],
-                                    region=region,
-                                )
                             )
-
-                        request = self.client.subnetworks().list_next(
-                            previous_request=request, previous_response=response
                         )
+
+                    request = self.client.subnetworks().list_next(
+                        previous_request=request, previous_response=response
+                    )
             except Exception as error:
                 logger.error(
                     f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"

prowler/providers/gcp/services/dataproc/dataproc_service.py

@@ -9,42 +9,40 @@ from prowler.providers.gcp.services.compute.compute_client import compute_client
 class Dataproc(GCPService):
     def __init__(self, audit_info):
         super().__init__(__class__.__name__, audit_info)
+        self.regions = compute_client.regions
         self.clusters = []
-        self.__get_clusters__()
+        self.__threading_call__(self.__get_clusters__, self.regions)
 
-    def __get_clusters__(self):
+    def __get_clusters__(self, region):
         for project_id in self.project_ids:
             try:
-                for region in compute_client.regions:
+                request = (
+                    self.client.projects()
+                    .regions()
+                    .clusters()
+                    .list(projectId=project_id, region=region)
+                )
+                while request is not None:
+                    response = request.execute(
+                        http=self.__get_AuthorizedHttp_client__()
+                    )
+
+                    for cluster in response.get("clusters", []):
+                        self.clusters.append(
+                            Cluster(
+                                name=cluster["clusterName"],
+                                id=cluster["clusterUuid"],
+                                encryption_config=cluster["config"]["encryptionConfig"],
+                                project_id=project_id,
+                            )
+                        )
+
                     request = (
                         self.client.projects()
                         .regions()
                         .clusters()
-                        .list(projectId=project_id, region=region)
+                        .list_next(previous_request=request, previous_response=response)
                     )
-                    while request is not None:
-                        response = request.execute()
-
-                        for cluster in response.get("clusters", []):
-                            self.clusters.append(
-                                Cluster(
-                                    name=cluster["clusterName"],
-                                    id=cluster["clusterUuid"],
-                                    encryption_config=cluster["config"][
-                                        "encryptionConfig"
-                                    ],
-                                    project_id=project_id,
-                                )
-                            )
-
-                        request = (
-                            self.client.projects()
-                            .regions()
-                            .clusters()
-                            .list_next(
-                                previous_request=request, previous_response=response
-                            )
-                        )
             except Exception as error:
                 logger.error(
                     f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"

prowler/providers/gcp/services/kms/kms_service.py

@@ -14,7 +14,7 @@ class KMS(GCPService):
         self.key_rings = []
         self.crypto_keys = []
         self.__get_locations__()
-        self.__get_key_rings__()
+        self.__threading_call__(self.__get_key_rings__, self.locations)
         self.__get_crypto_keys__()
         self.__get_crypto_keys_iam_policy__()
 
@@ -44,36 +44,32 @@ class KMS(GCPService):
                     f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
                 )
 
-    def __get_key_rings__(self):
+    def __get_key_rings__(self, location):
-        for location in self.locations:
+        try:
-            try:
+            request = (
+                self.client.projects().locations().keyRings().list(parent=location.name)
+            )
+            while request is not None:
+                response = request.execute(http=self.__get_AuthorizedHttp_client__())
+
+                for ring in response.get("keyRings", []):
+                    self.key_rings.append(
+                        KeyRing(
+                            name=ring["name"],
+                            project_id=location.project_id,
+                        )
+                    )
+
                 request = (
                     self.client.projects()
                     .locations()
                     .keyRings()
-                    .list(parent=location.name)
+                    .list_next(previous_request=request, previous_response=response)
-                )
-                while request is not None:
-                    response = request.execute()
-
-                    for ring in response.get("keyRings", []):
-                        self.key_rings.append(
-                            KeyRing(
-                                name=ring["name"],
-                                project_id=location.project_id,
-                            )
-                        )
-
-                    request = (
-                        self.client.projects()
-                        .locations()
-                        .keyRings()
-                        .list_next(previous_request=request, previous_response=response)
-                    )
-            except Exception as error:
-                logger.error(
-                    f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
                 )
+        except Exception as error:
+            logger.error(
+                f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
+            )
 
     def __get_crypto_keys__(self):
         for ring in self.key_rings:

pyproject.toml

@@ -38,6 +38,7 @@ botocore = "1.29.165"
 colorama = "0.4.6"
 detect-secrets = "1.4.0"
 google-api-python-client = "2.95.0"
+google-auth-httplib2 = "^0.1.0"
 mkdocs = {version = "1.4.3", optional = true}
 mkdocs-material = {version = "9.1.20", optional = true}
 msgraph-core = "0.2.2"