mirror of
https://github.com/ghndrx/prowler.git
synced 2026-02-10 06:45:08 +00:00
feat(refresh_aws_regions): Auto refresh of AWS regions for services. (#1221)
* feat(refresh_aws_regions): Auto refresh of AWS regions for services. * Update refresh_aws_services_regions.yml * Delete aws_regions_by_service.json * Update refresh_aws_services_regions.yml Co-authored-by: sergargar <sergio@verica.io>
This commit is contained in:
Sergio Garcia
50
.github/workflows/refresh_aws_services_regions.yml
vendored
Normal file
50
.github/workflows/refresh_aws_services_regions.yml
vendored
Normal file
@@ -0,0 +1,50 @@
|
||||
# This is a basic workflow to help you get started with Actions
|
||||
|
||||
name: Refresh regions of AWS services
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 9 * * *" #runs at 09:00 UTC everyday
|
||||
|
||||
env:
|
||||
GITHUB_BRANCH: "prowler-3.0-dev"
|
||||
|
||||
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
||||
jobs:
|
||||
# This workflow contains a single job called "build"
|
||||
build:
|
||||
# The type of runner that the job will run on
|
||||
runs-on: ubuntu-latest
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ env.GITHUB_BRANCH }}
|
||||
|
||||
- name: setup python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9 #install the python needed
|
||||
|
||||
# Runs a single command using the runners shell
|
||||
- name: Run a one-line script
|
||||
run: python3 util/update_aws_services_regions.py
|
||||
|
||||
# Create pull request
|
||||
- name: Create Pull Request
|
||||
uses: peter-evans/create-pull-request@v4
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
commit-message: "feat(regions_update): Update regions for AWS services."
|
||||
branch: "aws-services-regions-updated"
|
||||
labels: "status/waiting-for-revision, severity/low"
|
||||
title: "feat(regions_update): Changes in regions for AWS services."
|
||||
body: |
|
||||
### Description
|
||||
|
||||
This PR updates the regions for AWS services.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -7,7 +7,4 @@ prowler_version = "3.0-alfa"
|
||||
groups_file = "groups.json"
|
||||
|
||||
# AWS services-regions matrix json
|
||||
aws_services_json_url = (
|
||||
"https://api.regional-table.region-services.aws.a2z.com/index.json"
|
||||
)
|
||||
aws_services_json_file = "providers/aws/aws_regions_services.json"
|
||||
aws_services_json_file = "providers/aws/aws_regions_by_service.json"
|
||||
|
||||
@@ -156,7 +156,7 @@ def set_output_options(quiet):
|
||||
|
||||
def run_check(check):
|
||||
print(
|
||||
f"\nCheck Name: {check.checkName} - {Fore.MAGENTA}{check.serviceName}{Fore.YELLOW}[{check.severity}]{Style.RESET_ALL}"
|
||||
f"\nCheck Name: {check.checkName} - {Fore.MAGENTA}{check.serviceName}{Fore.YELLOW} [{check.severity}]{Style.RESET_ALL}"
|
||||
)
|
||||
logger.debug(f"Executing check: {check.checkName}")
|
||||
findings = check.execute()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,6 @@
|
||||
import json
|
||||
import threading
|
||||
import urllib.request
|
||||
|
||||
from config.config import aws_services_json_file, aws_services_json_url
|
||||
from config.config import aws_services_json_file
|
||||
from lib.logger import logger
|
||||
from lib.utils.utils import open_file, parse_json_file
|
||||
from providers.aws.aws_provider import current_audit_info
|
||||
@@ -24,63 +22,24 @@ class EC2:
|
||||
|
||||
def __generate_regional_clients__(self, service, audit_info):
|
||||
regional_clients = []
|
||||
try: # Try to get the list online
|
||||
with urllib.request.urlopen(aws_services_json_url) as url:
|
||||
data = json.loads(url.read().decode())
|
||||
except:
|
||||
# Get the list locally
|
||||
f = open_file(aws_services_json_file)
|
||||
data = parse_json_file(f)
|
||||
|
||||
for att in data["prices"]:
|
||||
if (
|
||||
audit_info.audited_regions
|
||||
): # Check for input aws audit_info.audited_regions
|
||||
if (
|
||||
service in att["id"].split(":")[0]
|
||||
and att["attributes"]["aws:region"] in audit_info.audited_regions
|
||||
): # Check if service has this region
|
||||
region = att["attributes"]["aws:region"]
|
||||
regional_client = audit_info.audit_session.client(
|
||||
service, region_name=region
|
||||
)
|
||||
regional_client.region = region
|
||||
regional_clients.append(regional_client)
|
||||
else:
|
||||
if audit_info.audited_partition in "aws":
|
||||
if (
|
||||
service in att["id"].split(":")[0]
|
||||
and "gov" not in att["attributes"]["aws:region"]
|
||||
and "cn" not in att["attributes"]["aws:region"]
|
||||
):
|
||||
region = att["attributes"]["aws:region"]
|
||||
regional_client = audit_info.audit_session.client(
|
||||
service, region_name=region
|
||||
)
|
||||
regional_client.region = region
|
||||
regional_clients.append(regional_client)
|
||||
elif audit_info.audited_partition in "cn":
|
||||
if (
|
||||
service in att["id"].split(":")[0]
|
||||
and "cn" in att["attributes"]["aws:region"]
|
||||
):
|
||||
region = att["attributes"]["aws:region"]
|
||||
regional_client = audit_info.audit_session.client(
|
||||
service, region_name=region
|
||||
)
|
||||
regional_client.region = region
|
||||
regional_clients.append(regional_client)
|
||||
elif audit_info.audited_partition in "gov":
|
||||
if (
|
||||
service in att["id"].split(":")[0]
|
||||
and "gov" in att["attributes"]["aws:region"]
|
||||
):
|
||||
region = att["attributes"]["aws:region"]
|
||||
regional_client = audit_info.audit_session.client(
|
||||
service, region_name=region
|
||||
)
|
||||
regional_client.region = region
|
||||
regional_clients.append(regional_client)
|
||||
# Get json locally
|
||||
f = open_file(aws_services_json_file)
|
||||
data = parse_json_file(f)
|
||||
json_regions = data["services"][service]["regions"][
|
||||
audit_info.audited_partition
|
||||
]
|
||||
if audit_info.audited_regions: # Check for input aws audit_info.audited_regions
|
||||
regions = list(
|
||||
set(json_regions).intersection(audit_info.audited_regions)
|
||||
) # Get common regions between input and json
|
||||
else: # Get all regions from json of the service and partition
|
||||
regions = json_regions
|
||||
for region in regions:
|
||||
regional_client = audit_info.audit_session.client(
|
||||
service, region_name=region
|
||||
)
|
||||
regional_client.region = region
|
||||
regional_clients.append(regional_client)
|
||||
|
||||
return regional_clients
|
||||
|
||||
|
||||
72
util/update_aws_services_regions.py
Normal file
72
util/update_aws_services_regions.py
Normal file
@@ -0,0 +1,72 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from urllib import request
|
||||
|
||||
aws_services_json_url = (
|
||||
"https://api.regional-table.region-services.aws.a2z.com/index.json"
|
||||
)
|
||||
|
||||
# Logging config
|
||||
logging.basicConfig(
|
||||
stream=sys.stdout,
|
||||
format="%(asctime)s [File: %(filename)s:%(lineno)d] \t[Module: %(module)s]\t %(levelname)s: %(message)s",
|
||||
datefmt="%m/%d/%Y %I:%M:%S %p",
|
||||
level=logging.INFO,
|
||||
)
|
||||
|
||||
# JSON files
|
||||
with request.urlopen(aws_services_json_url) as url: # Get the AWS regions matrix online
|
||||
logging.info(f"Downloading JSON from {aws_services_json_url}")
|
||||
original_matrix_regions_aws = json.loads(url.read().decode())
|
||||
parsed__matrix_regions_aws = f"{os.path.dirname(os.path.realpath(__name__))}/providers/aws/aws_regions_by_service.json"
|
||||
|
||||
# JSON objects
|
||||
regions_by_service = {}
|
||||
services = {}
|
||||
old_service = ""
|
||||
|
||||
logging.info("Recovering AWS Regions by Service")
|
||||
# Iterating through the json list
|
||||
for item in original_matrix_regions_aws["prices"]:
|
||||
service = item["id"].split(":")[0]
|
||||
region = item["id"].split(":")[1]
|
||||
# Init regions for the new service
|
||||
if service != old_service:
|
||||
regions_dict = {}
|
||||
# Store the service
|
||||
services[service] = regions_dict
|
||||
# Init objects for every new service
|
||||
old_service = service
|
||||
regions = {}
|
||||
regions["aws"] = {}
|
||||
regions["cn"] = {}
|
||||
regions["gov"] = {}
|
||||
regions_dict["regions"] = {}
|
||||
regions_aws = []
|
||||
regions_cn = []
|
||||
regions_gov = []
|
||||
|
||||
# Include the region in their AWS partition
|
||||
if "cn-" in region:
|
||||
regions_cn.append(region)
|
||||
regions["cn"] = regions_cn
|
||||
|
||||
elif "gov-" in region:
|
||||
regions_gov.append(region)
|
||||
regions["gov"] = regions_gov
|
||||
else:
|
||||
regions_aws.append(region)
|
||||
regions["aws"] = regions_aws
|
||||
|
||||
regions_dict["regions"] = regions
|
||||
|
||||
# Store final JSON
|
||||
logging.info(f"Storing final JSON")
|
||||
regions_by_service["services"] = services
|
||||
|
||||
# Write to file
|
||||
logging.info(f"Writing {parsed__matrix_regions_aws}")
|
||||
with open(parsed__matrix_regions_aws, "w") as outfile:
|
||||
json.dump(regions_by_service, outfile, indent=2, sort_keys=True)
|
||||
Reference in New Issue
Block a user