feat(refresh_aws_regions): Auto refresh of AWS regions for services. (#1221)

* feat(refresh_aws_regions): Auto refresh of AWS regions for services.

* Update refresh_aws_services_regions.yml

* Delete aws_regions_by_service.json

* Update refresh_aws_services_regions.yml

Co-authored-by: sergargar <sergio@verica.io>

.github/workflows/refresh_aws_services_regions.yml

@@ -0,0 +1,50 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Refresh regions of AWS services
+
+on:
+  schedule:
+    - cron: "0 9 * * *" #runs at 09:00 UTC everyday
+
+env:
+  GITHUB_BRANCH: "prowler-3.0-dev"
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.GITHUB_BRANCH }}
+
+      - name: setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9 #install the python needed
+
+      # Runs a single command using the runners shell
+      - name: Run a one-line script
+        run: python3 util/update_aws_services_regions.py
+
+      # Create pull request
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "feat(regions_update): Update regions for AWS services."
+          branch: "aws-services-regions-updated"
+          labels: "status/waiting-for-revision, severity/low"
+          title: "feat(regions_update): Changes in regions for AWS services."
+          body: |
+            ### Description
+
+            This PR updates the regions for AWS services.
+            
+            ### License
+
+            By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

config/config.py

@@ -7,7 +7,4 @@ prowler_version = "3.0-alfa"
 groups_file = "groups.json"
 
 # AWS services-regions matrix json
-aws_services_json_url = (
+aws_services_json_file = "providers/aws/aws_regions_by_service.json"
-    "https://api.regional-table.region-services.aws.a2z.com/index.json"
-)
-aws_services_json_file = "providers/aws/aws_regions_services.json"

providers/aws/services/ec2/ec2_service.py

@@ -1,8 +1,6 @@
-import json
 import threading
-import urllib.request
 
-from config.config import aws_services_json_file, aws_services_json_url
+from config.config import aws_services_json_file
 from lib.logger import logger
 from lib.utils.utils import open_file, parse_json_file
 from providers.aws.aws_provider import current_audit_info
@@ -24,58 +22,19 @@ class EC2:
 
     def __generate_regional_clients__(self, service, audit_info):
         regional_clients = []
-        try:  # Try to get the list online
+        # Get json locally
-            with urllib.request.urlopen(aws_services_json_url) as url:
-                data = json.loads(url.read().decode())
-        except:
-            # Get the list locally
         f = open_file(aws_services_json_file)
         data = parse_json_file(f)
-
+        json_regions = data["services"][service]["regions"][
-        for att in data["prices"]:
+            audit_info.audited_partition
-            if (
+        ]
-                audit_info.audited_regions
+        if audit_info.audited_regions:  # Check for input aws audit_info.audited_regions
-            ):  # Check for input aws audit_info.audited_regions
+            regions = list(
-                if (
+                set(json_regions).intersection(audit_info.audited_regions)
-                    service in att["id"].split(":")[0]
+            )  # Get common regions between input and json
-                    and att["attributes"]["aws:region"] in audit_info.audited_regions
+        else:  # Get all regions from json of the service and partition
-                ):  # Check if service has this region
+            regions = json_regions
-                    region = att["attributes"]["aws:region"]
+        for region in regions:
-                    regional_client = audit_info.audit_session.client(
-                        service, region_name=region
-                    )
-                    regional_client.region = region
-                    regional_clients.append(regional_client)
-            else:
-                if audit_info.audited_partition in "aws":
-                    if (
-                        service in att["id"].split(":")[0]
-                        and "gov" not in att["attributes"]["aws:region"]
-                        and "cn" not in att["attributes"]["aws:region"]
-                    ):
-                        region = att["attributes"]["aws:region"]
-                        regional_client = audit_info.audit_session.client(
-                            service, region_name=region
-                        )
-                        regional_client.region = region
-                        regional_clients.append(regional_client)
-                elif audit_info.audited_partition in "cn":
-                    if (
-                        service in att["id"].split(":")[0]
-                        and "cn" in att["attributes"]["aws:region"]
-                    ):
-                        region = att["attributes"]["aws:region"]
-                        regional_client = audit_info.audit_session.client(
-                            service, region_name=region
-                        )
-                        regional_client.region = region
-                        regional_clients.append(regional_client)
-                elif audit_info.audited_partition in "gov":
-                    if (
-                        service in att["id"].split(":")[0]
-                        and "gov" in att["attributes"]["aws:region"]
-                    ):
-                        region = att["attributes"]["aws:region"]
             regional_client = audit_info.audit_session.client(
                 service, region_name=region
             )

util/update_aws_services_regions.py

@@ -0,0 +1,72 @@
+import json
+import logging
+import os
+import sys
+from urllib import request
+
+aws_services_json_url = (
+    "https://api.regional-table.region-services.aws.a2z.com/index.json"
+)
+
+# Logging config
+logging.basicConfig(
+    stream=sys.stdout,
+    format="%(asctime)s [File: %(filename)s:%(lineno)d] \t[Module: %(module)s]\t %(levelname)s: %(message)s",
+    datefmt="%m/%d/%Y %I:%M:%S %p",
+    level=logging.INFO,
+)
+
+# JSON files
+with request.urlopen(aws_services_json_url) as url:  # Get the AWS regions matrix online
+    logging.info(f"Downloading JSON from {aws_services_json_url}")
+    original_matrix_regions_aws = json.loads(url.read().decode())
+parsed__matrix_regions_aws = f"{os.path.dirname(os.path.realpath(__name__))}/providers/aws/aws_regions_by_service.json"
+
+# JSON objects
+regions_by_service = {}
+services = {}
+old_service = ""
+
+logging.info("Recovering AWS Regions by Service")
+# Iterating through the json list
+for item in original_matrix_regions_aws["prices"]:
+    service = item["id"].split(":")[0]
+    region = item["id"].split(":")[1]
+    # Init regions for the new service
+    if service != old_service:
+        regions_dict = {}
+        # Store the service
+        services[service] = regions_dict
+        # Init objects for every new service
+        old_service = service
+        regions = {}
+        regions["aws"] = {}
+        regions["cn"] = {}
+        regions["gov"] = {}
+        regions_dict["regions"] = {}
+        regions_aws = []
+        regions_cn = []
+        regions_gov = []
+
+    # Include the region in their AWS partition
+    if "cn-" in region:
+        regions_cn.append(region)
+        regions["cn"] = regions_cn
+
+    elif "gov-" in region:
+        regions_gov.append(region)
+        regions["gov"] = regions_gov
+    else:
+        regions_aws.append(region)
+        regions["aws"] = regions_aws
+
+    regions_dict["regions"] = regions
+
+# Store final JSON
+logging.info(f"Storing final JSON")
+regions_by_service["services"] = services
+
+# Write to file
+logging.info(f"Writing {parsed__matrix_regions_aws}")
+with open(parsed__matrix_regions_aws, "w") as outfile:
+    json.dump(regions_by_service, outfile, indent=2, sort_keys=True)