ghndrx/prowler
1
0
Fork 0
mirror of https://github.com/ghndrx/prowler.git synced 2026-02-10 06:45:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(gcp): Improve gcp performance (#2662)

Browse Source
This commit is contained in:
Sergio Garcia
2023-08-03 10:52:52 +02:00
committed by GitHub
parent efa75a62e3
commit 2d832bca15
8 changed files with 165 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
poetry.lock generated
View File

@@ -2383,8 +2383,7 @@ files = [
{file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"},
{file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"},
{file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"},
{file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"},
{file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"},
{file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_12_6_arm64.whl", hash = "sha256:721bc4ba4525f53f6a611ec0967bdcee61b31df5a56801281027a3a6d1c2daf5"},
{file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"},
{file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"},
{file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"},
@@ -2876,4 +2875,4 @@ docs = ["mkdocs", "mkdocs-material"]
[metadata]
lock-version = "2.0"
python-versions = "^3.9"
content-hash = "95a3c66d752dc8552a3e2c032545cda76f13f056ff0f0554a8664f20c8da39c4"
content-hash = "2e4af8e15db5d10b860a960d9fa3f4511182f858121e326f98aeca4a1bd75c86"

4
prowler/providers/gcp/gcp_provider.py
View File

@@ -47,7 +47,9 @@ class GCP_Provider:
if credentials_file:
self.__set_gcp_creds_env_var__(credentials_file)
return auth.default()
return auth.default(
scopes=["https://www.googleapis.com/auth/cloud-platform"]
)
except Exception as error:
logger.critical(
f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"

48
prowler/providers/gcp/lib/service/service.py
View File

@@ -1,3 +1,10 @@
import threading
import google_auth_httplib2
import httplib2
from googleapiclient import discovery
from prowler.lib.logger import logger
from prowler.providers.gcp.gcp_provider import generate_client
@@ -6,13 +13,48 @@ class GCPService:
# We receive the service using __class__.__name__ or the service name in lowercase
# e.g.: APIKeys --> we need a lowercase string, so service.lower()
self.service = service.lower() if not service.islower() else service
self.credentials = audit_info.credentials
self.api_version = api_version
self.project_ids = audit_info.project_ids
self.default_project_id = audit_info.default_project_id
self.region = region
self.client = generate_client(service, api_version, audit_info)
# Only project ids that have their API enabled will be scanned
self.project_ids = self.__is_api_active__(audit_info.project_ids)
def __get_client__(self):
return self.client
def __threading_call__(self, call, iterator):
threads = []
for value in iterator:
threads.append(threading.Thread(target=call, args=(value,)))
for t in threads:
t.start()
for t in threads:
t.join()
def __get_AuthorizedHttp_client__(self):
return google_auth_httplib2.AuthorizedHttp(
self.credentials, http=httplib2.Http()
)
def __is_api_active__(self, audited_project_ids):
project_ids = []
for project_id in audited_project_ids:
try:
client = discovery.build("serviceusage", "v1")
request = client.services().get(
name=f"projects/{project_id}/services/{self.service}.googleapis.com"
)
response = request.execute()
if response.get("state") != "DISABLED":
project_ids.append(project_id)
else:
logger.error(
f"{self.service} API has not been used in project {project_id} before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/dataproc.googleapis.com/overview?project={project_id} then retry."
)
except Exception as error:
logger.error(
f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
)
return project_ids

2
prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py
View File

@@ -43,7 +43,7 @@ class CloudResourceManager(GCPService):
def __get_organizations__(self):
try:
response = self.client.organizations().search().execute()
for org in response["organizations"]:
for org in response.get("organizations", []):
self.organizations.append(
Organization(id=org["name"].split("/")[-1], name=org["displayName"])
)

134
prowler/providers/gcp/services/compute/compute_service.py
View File

@@ -21,9 +21,9 @@ class Compute(GCPService):
self.__get_regions__()
self.__get_projects__()
self.__get_zones__()
self.__get_instances__()
self.__threading_call__(self.__get_instances__, self.zones)
self.__get_networks__()
self.__get_subnetworks__()
self.__threading_call__(self.__get_subnetworks__, self.regions)
self.__get_firewalls__()
def __get_regions__(self):
@@ -78,60 +78,57 @@ class Compute(GCPService):
f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
)
def __get_instances__(self):
def __get_instances__(self, zone):
for project_id in self.project_ids:
try:
for zone in self.zones:
request = self.client.instances().list(
project=project_id, zone=zone
request = self.client.instances().list(project=project_id, zone=zone)
while request is not None:
response = request.execute(
http=self.__get_AuthorizedHttp_client__()
)
while request is not None:
response = request.execute()
for instance in response.get("items", []):
public_ip = False
for interface in instance["networkInterfaces"]:
for config in interface.get("accessConfigs", []):
if "natIP" in config:
public_ip = True
self.instances.append(
Instance(
name=instance["name"],
id=instance["id"],
zone=zone,
public_ip=public_ip,
metadata=instance["metadata"],
shielded_enabled_vtpm=instance[
"shieldedInstanceConfig"
]["enableVtpm"],
shielded_enabled_integrity_monitoring=instance[
"shieldedInstanceConfig"
]["enableIntegrityMonitoring"],
confidential_computing=instance.get(
"confidentialInstanceConfig", {}
).get("enableConfidentialCompute", False),
service_accounts=instance.get(
"serviceAccounts", []
),
ip_forward=instance.get("canIpForward", False),
disks_encryption=[
(
disk["deviceName"],
True
if disk.get("diskEncryptionKey", {}).get(
"sha256"
)
else False,
for instance in response.get("items", []):
public_ip = False
for interface in instance["networkInterfaces"]:
for config in interface.get("accessConfigs", []):
if "natIP" in config:
public_ip = True
self.instances.append(
Instance(
name=instance["name"],
id=instance["id"],
zone=zone,
public_ip=public_ip,
metadata=instance["metadata"],
shielded_enabled_vtpm=instance[
"shieldedInstanceConfig"
]["enableVtpm"],
shielded_enabled_integrity_monitoring=instance[
"shieldedInstanceConfig"
]["enableIntegrityMonitoring"],
confidential_computing=instance.get(
"confidentialInstanceConfig", {}
).get("enableConfidentialCompute", False),
service_accounts=instance.get("serviceAccounts", []),
ip_forward=instance.get("canIpForward", False),
disks_encryption=[
(
disk["deviceName"],
True
if disk.get("diskEncryptionKey", {}).get(
"sha256"
)
for disk in instance["disks"]
],
project_id=project_id,
)
else False,
)
for disk in instance["disks"]
],
project_id=project_id,
)
request = self.client.instances().list_next(
previous_request=request, previous_response=response
)
request = self.client.instances().list_next(
previous_request=request, previous_response=response
)
except Exception as error:
logger.error(
f"{zone} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
@@ -168,30 +165,31 @@ class Compute(GCPService):
f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
)
def __get_subnetworks__(self):
def __get_subnetworks__(self, region):
for project_id in self.project_ids:
try:
for region in self.regions:
request = self.client.subnetworks().list(
project=project_id, region=region
request = self.client.subnetworks().list(
project=project_id, region=region
)
while request is not None:
response = request.execute(
http=self.__get_AuthorizedHttp_client__()
)
while request is not None:
response = request.execute()
for subnet in response.get("items", []):
self.subnets.append(
Subnet(
name=subnet["name"],
id=subnet["id"],
project_id=project_id,
flow_logs=subnet.get("enableFlowLogs", False),
network=subnet["network"].split("/")[-1],
region=region,
)
for subnet in response.get("items", []):
self.subnets.append(
Subnet(
name=subnet["name"],
id=subnet["id"],
project_id=project_id,
flow_logs=subnet.get("enableFlowLogs", False),
network=subnet["network"].split("/")[-1],
region=region,
)
request = self.client.subnetworks().list_next(
previous_request=request, previous_response=response
)
request = self.client.subnetworks().list_next(
previous_request=request, previous_response=response
)
except Exception as error:
logger.error(
f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"

52
prowler/providers/gcp/services/dataproc/dataproc_service.py
View File

@@ -9,42 +9,40 @@ from prowler.providers.gcp.services.compute.compute_client import compute_client
class Dataproc(GCPService):
def __init__(self, audit_info):
super().__init__(__class__.__name__, audit_info)
self.regions = compute_client.regions
self.clusters = []
self.__get_clusters__()
self.__threading_call__(self.__get_clusters__, self.regions)
def __get_clusters__(self):
def __get_clusters__(self, region):
for project_id in self.project_ids:
try:
for region in compute_client.regions:
request = (
self.client.projects()
.regions()
.clusters()
.list(projectId=project_id, region=region)
)
while request is not None:
response = request.execute(
http=self.__get_AuthorizedHttp_client__()
)
for cluster in response.get("clusters", []):
self.clusters.append(
Cluster(
name=cluster["clusterName"],
id=cluster["clusterUuid"],
encryption_config=cluster["config"]["encryptionConfig"],
project_id=project_id,
)
)
request = (
self.client.projects()
.regions()
.clusters()
.list(projectId=project_id, region=region)
.list_next(previous_request=request, previous_response=response)
)
while request is not None:
response = request.execute()
for cluster in response.get("clusters", []):
self.clusters.append(
Cluster(
name=cluster["clusterName"],
id=cluster["clusterUuid"],
encryption_config=cluster["config"][
"encryptionConfig"
],
project_id=project_id,
)
)
request = (
self.client.projects()
.regions()
.clusters()
.list_next(
previous_request=request, previous_response=response
)
)
except Exception as error:
logger.error(
f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"

48
prowler/providers/gcp/services/kms/kms_service.py
View File

@@ -14,7 +14,7 @@ class KMS(GCPService):
self.key_rings = []
self.crypto_keys = []
self.__get_locations__()
self.__get_key_rings__()
self.__threading_call__(self.__get_key_rings__, self.locations)
self.__get_crypto_keys__()
self.__get_crypto_keys_iam_policy__()
@@ -44,36 +44,32 @@ class KMS(GCPService):
f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
)
def __get_key_rings__(self):
for location in self.locations:
try:
def __get_key_rings__(self, location):
try:
request = (
self.client.projects().locations().keyRings().list(parent=location.name)
)
while request is not None:
response = request.execute(http=self.__get_AuthorizedHttp_client__())
for ring in response.get("keyRings", []):
self.key_rings.append(
KeyRing(
name=ring["name"],
project_id=location.project_id,
)
)
request = (
self.client.projects()
.locations()
.keyRings()
.list(parent=location.name)
)
while request is not None:
response = request.execute()
for ring in response.get("keyRings", []):
self.key_rings.append(
KeyRing(
name=ring["name"],
project_id=location.project_id,
)
)
request = (
self.client.projects()
.locations()
.keyRings()
.list_next(previous_request=request, previous_response=response)
)
except Exception as error:
logger.error(
f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
.list_next(previous_request=request, previous_response=response)
)
except Exception as error:
logger.error(
f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
)
def __get_crypto_keys__(self):
for ring in self.key_rings:

1
pyproject.toml
View File

@@ -38,6 +38,7 @@ botocore = "1.29.165"
colorama = "0.4.6"
detect-secrets = "1.4.0"
google-api-python-client = "2.95.0"
google-auth-httplib2 = "^0.1.0"
mkdocs = {version = "1.4.3", optional = true}
mkdocs-material = {version = "9.1.20", optional = true}
msgraph-core = "0.2.2"