From 24ca19d502aec1f445bc34344f9fd10699c9cf3f Mon Sep 17 00:00:00 2001 From: Pepe Fagoaga Date: Thu, 17 Nov 2022 11:31:20 +0100 Subject: [PATCH] feat(EMR): Service and checks (#1486) Co-authored-by: Sergio Garcia <38561120+sergargar@users.noreply.github.com> --- .../aws/services/ec2/lib/security_groups.py | 104 +++-- .../services/ec2/lib/security_groups_test.py | 21 + providers/aws/services/emr/__init__.py | 0 providers/aws/services/emr/check_extra7176 | 55 --- providers/aws/services/emr/check_extra7177 | 124 ------ providers/aws/services/emr/check_extra7178 | 41 -- providers/aws/services/emr/emr_client.py | 4 + .../__init__.py | 0 ...account_public_block_enabled.metadata.json | 35 ++ ...mr_cluster_account_public_block_enabled.py | 24 ++ ...uster_account_public_block_enabled_test.py | 67 ++++ .../__init__.py | 0 ...er_master_nodes_no_public_ip.metadata.json | 35 ++ .../emr_cluster_master_nodes_no_public_ip.py | 30 ++ ..._cluster_master_nodes_no_public_ip_test.py | 169 ++++++++ .../__init__.py | 0 ...r_cluster_publicly_accesible.metadata.json | 35 ++ .../emr_cluster_publicly_accesible.py | 86 +++++ .../emr_cluster_publicly_accesible_test.py | 361 ++++++++++++++++++ providers/aws/services/emr/emr_service.py | 166 ++++++++ .../aws/services/emr/emr_service_test.py | 136 +++++++ 21 files changed, 1249 insertions(+), 244 deletions(-) create mode 100644 providers/aws/services/ec2/lib/security_groups_test.py create mode 100644 providers/aws/services/emr/__init__.py delete mode 100644 providers/aws/services/emr/check_extra7176 delete mode 100644 providers/aws/services/emr/check_extra7177 delete mode 100644 providers/aws/services/emr/check_extra7178 create mode 100644 providers/aws/services/emr/emr_client.py create mode 100644 providers/aws/services/emr/emr_cluster_account_public_block_enabled/__init__.py create mode 100644 providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled.metadata.json create mode 100644 providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled.py create mode 100644 providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled_test.py create mode 100644 providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/__init__.py create mode 100644 providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip.metadata.json create mode 100644 providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip.py create mode 100644 providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip_test.py create mode 100644 providers/aws/services/emr/emr_cluster_publicly_accesible/__init__.py create mode 100644 providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible.metadata.json create mode 100644 providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible.py create mode 100644 providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible_test.py create mode 100644 providers/aws/services/emr/emr_service.py create mode 100644 providers/aws/services/emr/emr_service_test.py diff --git a/providers/aws/services/ec2/lib/security_groups.py b/providers/aws/services/ec2/lib/security_groups.py index 2ede4f79..564208e9 100644 --- a/providers/aws/services/ec2/lib/security_groups.py +++ b/providers/aws/services/ec2/lib/security_groups.py @@ -1,27 +1,48 @@ +import ipaddress from typing import Any ################## Security Groups -# Check if the security group ingress rule has public access to the check_ports using the protocol def check_security_group(ingress_rule: Any, protocol: str, ports: list = []) -> bool: - public_IPv4 = "0.0.0.0/0" - public_IPv6 = "::/0" + """ + Check if the security group ingress rule has public access to the check_ports using the protocol + + @param ingress_rule: AWS Security Group IpPermissions Ingress Rule + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + { + 'CidrIp': 'string', + 'Description': 'string' + }, + ], + 'Ipv6Ranges': [ + { + 'CidrIpv6': 'string', + 'Description': 'string' + }, + ], + 'ToPort': 123, + } + + @param procotol: Protocol to check. + + + @param ports: List of ports to check. (Default: []) + """ # Check for all traffic ingress rules regardless of the protocol - if ingress_rule["IpProtocol"] == "-1" and ( - ( - "0.0.0.0/0" in str(ingress_rule["IpRanges"]) - or "::/0" in str(ingress_rule["Ipv6Ranges"]) - ) - ): - return True + if ingress_rule["IpProtocol"] == "-1": + for ip_ingress_rule in ingress_rule["IpRanges"]: + if _is_cidr_public(ip_ingress_rule["CidrIp"]): + return True + for ip_ingress_rule in ingress_rule["Ipv6Ranges"]: + if _is_cidr_public(ip_ingress_rule["CidrIp"]): + return True # Check for specific ports in ingress rules if "FromPort" in ingress_rule: - # All ports - if ingress_rule["FromPort"] == 0 and ingress_rule["ToPort"] == 65535: - return True - # If there is a port range if ingress_rule["FromPort"] != ingress_rule["ToPort"]: # Calculate port range, adding 1 @@ -35,14 +56,49 @@ def check_security_group(ingress_rule: Any, protocol: str, ports: list = []) -> ingress_port_range.append(int(ingress_rule["FromPort"])) # Test Security Group - for port in ports: - if ( - ( - public_IPv4 in str(ingress_rule["IpRanges"]) - or public_IPv6 in str(ingress_rule["Ipv6Ranges"]) - ) - and port in ingress_port_range - and ingress_rule["IpProtocol"] == protocol - ): - return True + # IPv4 + for ip_ingress_rule in ingress_rule["IpRanges"]: + if _is_cidr_public(ip_ingress_rule["CidrIp"]): + # If there are input ports to check + if ports: + for port in ports: + if ( + port in ingress_port_range + and ingress_rule["IpProtocol"] == protocol + ): + return True + else: + return True + + # IPv6 + for ip_ingress_rule in ingress_rule["Ipv6Ranges"]: + if _is_cidr_public(ip_ingress_rule["CidrIp"]): + # If there are input ports to check + if ports: + for port in ports: + if ( + port in ingress_port_range + and ingress_rule["IpProtocol"] == protocol + ): + return True + else: + return True + return False + + +def _is_cidr_public(cidr: str) -> bool: + """ + Check if an input CIDR is public + + @param cidr: CIDR 10.22.33.44/8 + """ + public_IPv4 = "0.0.0.0/0" + public_IPv6 = "::/0" + # Workaround until this issue is fixed + # PR https://github.com/python/cpython/pull/97733 + # Issue https://github.com/python/cpython/issues/82836 + if cidr in (public_IPv4, public_IPv6): + return True + + return ipaddress.ip_network(cidr).is_global diff --git a/providers/aws/services/ec2/lib/security_groups_test.py b/providers/aws/services/ec2/lib/security_groups_test.py new file mode 100644 index 00000000..0dd63cbb --- /dev/null +++ b/providers/aws/services/ec2/lib/security_groups_test.py @@ -0,0 +1,21 @@ +import pytest + +from providers.aws.services.ec2.lib.security_groups import _is_cidr_public + + +class Test_security_groups: + def test__is_cidr_public_Public_IP(self): + cidr = "0.0.0.0/0" + assert _is_cidr_public(cidr) + + def test__is_cidr_public_Private_IP(self): + cidr = "10.0.0.0/8" + assert not _is_cidr_public(cidr) + + def test__is_cidr_public_Bad_Private_IP(self): + cidr = "10.0.0.0/0" + with pytest.raises(ValueError) as ex: + _is_cidr_public(cidr) + + assert ex.type == ValueError + assert ex.match(f"{cidr} has host bits set") diff --git a/providers/aws/services/emr/__init__.py b/providers/aws/services/emr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/aws/services/emr/check_extra7176 b/providers/aws/services/emr/check_extra7176 deleted file mode 100644 index f4c5ce82..00000000 --- a/providers/aws/services/emr/check_extra7176 +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. - -CHECK_ID_extra7176="7.176" -CHECK_TITLE_extra7176="[extra7176] EMR Cluster without Public IP" -CHECK_SCORED_extra7176="NOT_SCORED" -CHECK_CIS_LEVEL_extra7176="EXTRA" -CHECK_SEVERITY_extra7176="Medium" -CHECK_ASFF_TYPE_extra7176="AwsEMR" -CHECK_ALTERNATE_check7176="extra7176" -CHECK_SERVICENAME_extra7176="emr" -CHECK_RISK_extra7176='EMR Cluster should not have Public IP' -CHECK_REMEDIATION_extra7176='Only make acceptable EMR clusters public' -CHECK_DOC_extra7176='https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html' -CHECK_CAF_EPIC_extra7176='Infrastructure Security' - -extra7176(){ - # Public EMR cluster have their DNS ending with .amazonaws.com while private ones have format of ip-xxx-xx-xx.us-east-1.compute.internal. - for regx in ${REGIONS}; do - # List only EMR clusters with the following states: STARTING, BOOTSTRAPPING, RUNNING, WAITING, TERMINATING - # [NOT TERMINATED AND TERMINATED_WITH_ERRORS] - LIST_OF_CLUSTERS=$("${AWSCLI}" emr list-clusters ${PROFILE_OPT} --region "${regx}" --query 'Clusters[?(Status.State!=`TERMINATED` && Status.State!=`TERMINATED_WITH_ERRORS`)].Id' --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${LIST_OF_CLUSTERS}"; then - textInfo "${regx}: Access Denied trying to list clusters of emr" "${regx}" - continue - fi - if [[ "${LIST_OF_CLUSTERS}" ]] - then - for cluster_id in ${LIST_OF_CLUSTERS}; do - master_public_dns=$("${AWSCLI}" emr describe-cluster ${PROFILE_OPT} --cluster-id "${cluster_id}" --query 'Cluster.MasterPublicDnsName' --region "${regx}" --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_public_dns}"; then - textInfo "${regx}: Access Denied trying to describe emr cluster" "${regx}" "${cluster_id}" - continue - fi - if [[ $master_public_dns != None && $master_public_dns != *.internal ]];then - textFail "${regx}: EMR Cluster ${cluster_id} has a Public IP" "${regx}" "${cluster_id}" - else - textPass "${regx}: EMR Cluster ${cluster_id} has not a Public IP" "${regx}" "${cluster_id}" - fi - done - else - textInfo "${regx}: No EMR Clusters found" "${regx}" - fi - done -} diff --git a/providers/aws/services/emr/check_extra7177 b/providers/aws/services/emr/check_extra7177 deleted file mode 100644 index 54615ddb..00000000 --- a/providers/aws/services/emr/check_extra7177 +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash - -# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. - - -CHECK_ID_extra7177="7.177" -CHECK_TITLE_extra7177="[extra7177] Publicly accessible EMR Cluster" -CHECK_SCORED_extra7177="NOT_SCORED" -CHECK_CIS_LEVEL_extra7177="EXTRA" -CHECK_SEVERITY_extra7177="High" -CHECK_ASFF_TYPE_extra7177="AwsEMR" -CHECK_ALTERNATE_check7177="extra7177" -CHECK_SERVICENAME_extra7177="emr" -CHECK_RISK_extra7177='EMR Clusters should not be publicly accessible' -CHECK_REMEDIATION_extra7177='Only make acceptable EMR clusters public' -CHECK_DOC_extra7177='https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html' -CHECK_CAF_EPIC_extra7177='Infrastructure Security' - -extra7177(){ - for regx in ${REGIONS}; do - # List only EMR clusters with the following states: STARTING, BOOTSTRAPPING, RUNNING, WAITING, TERMINATING - # [NOT TERMINATED AND TERMINATED_WITH_ERRORS] - LIST_OF_CLUSTERS=$("${AWSCLI}" emr list-clusters ${PROFILE_OPT} --region "${regx}" --query 'Clusters[?(Status.State!=`TERMINATED` && Status.State!=`TERMINATED_WITH_ERRORS`)].Id' --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${LIST_OF_CLUSTERS}"; then - textInfo "${regx}: Access Denied trying to list EMR clusters" "${regx}" - continue - fi - if [[ "${LIST_OF_CLUSTERS}" ]] - then - for cluster_id in ${LIST_OF_CLUSTERS}; do - master_public_dns=$("${AWSCLI}" emr describe-cluster ${PROFILE_OPT} --cluster-id "${cluster_id}" --query 'Cluster.MasterPublicDnsName' --region "${regx}" --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_public_dns}"; then - textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}" - continue - fi - if [[ $master_public_dns != None && $master_public_dns != *.internal ]];then - # If EMR cluster is Public, it is required to check their Security Groups for the Master, the Slaves and the additional ones - - # Retrive EMR Master Node Security Groups rules - master_node_sg=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.EmrManagedMasterSecurityGroup' --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_node_sg}"; then - textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}" - continue - fi - master_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${master_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_node_sg_internet_open}"; then - textInfo "$regx: Access Denied trying to describe security groups" "$regx" - continue - fi - - # Retrive EMR Slave Node Security Groups rules - slave_node_sg=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.EmrManagedSlaveSecurityGroup' --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg}"; then - textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}" - continue - fi - slave_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${slave_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg_internet_open}"; then - textInfo "$regx: Access Denied trying to describe security groups" "$regx" - continue - fi - - # Retrive EMR Additional Master node Security Groups rules - additional_master_node_sg_list=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.AdditionalMasterSecurityGroups' --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg}"; then - textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}" - continue - fi - local additional_master_node_sg_internet_open_list - if [[ "${additional_master_node_sg_list}" != "None" ]]; then - for additional_master_node_sg in ${additional_master_node_sg_list}; do - additional_master_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${additional_master_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg_internet_open}"; then - textInfo "$regx: Access Denied trying to describe security groups" "$regx" - continue - fi - # Store additional master node security groups that allows access from the internet - additional_master_node_sg_internet_open_list+=( "${additional_master_node_sg_internet_open}" ) - done - fi - - # Retrive EMR Additional Slave node Security Groups rules - additional_slave_node_sg_list=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.AdditionalSlaveSecurityGroups' --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg}"; then - textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}" - continue - fi - local additional_slave_node_sg_internet_open_list - if [[ "${additional_slave_node_sg_list}" != "None" ]]; then - for additional_slave_node_sg in ${additional_master_node_sg_list}; do - additional_slave_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${additional_slave_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${additional_slave_node_sg_internet_open}"; then - textInfo "$regx: Access Denied trying to describe security groups" "$regx" - continue - fi - # Store additional slave node security groups that allows access from the internet - additional_slave_node_sg_internet_open_list+=( "${additional_slave_node_sg_internet_open}" ) - done - fi - - # Check if EMR Cluster is publicly accessible through a Security Group - if [[ -n "${master_node_sg_internet_open}" || -n "${slave_node_sg_internet_open}" || "${#additional_master_node_sg_internet_open_list[@]}" -ne 0 || "${#additional_slave_node_sg_internet_open_list[@]}" -ne 0 ]]; then - textFail "${regx}: EMR Cluster ${cluster_id} is publicly accessible through the following Security Groups: Master Node ${master_node_sg_internet_open} ${additional_master_node_sg_internet_open_list[*]} -- Slaves Nodes ${slave_node_sg_internet_open} ${additional_slave_node_sg_internet_open_list[*]}" "${regx}" "${cluster_id}" - else - textPass "${regx}: EMR Cluster ${cluster_id} is not publicly accessible" "${regx}" "${cluster_id}" - fi - else - textPass "${regx}: EMR Cluster ${cluster_id} is not publicly accessible" "${regx}" "${cluster_id}" - fi - done - else - textInfo "${regx}: No EMR Clusters found" "${regx}" - fi - done -} diff --git a/providers/aws/services/emr/check_extra7178 b/providers/aws/services/emr/check_extra7178 deleted file mode 100644 index c86b12b2..00000000 --- a/providers/aws/services/emr/check_extra7178 +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. - - -CHECK_ID_extra7178="7.178" -CHECK_TITLE_extra7178="[extra7178] EMR Account Public Access Block enabled" -CHECK_SCORED_extra7178="NOT_SCORED" -CHECK_CIS_LEVEL_extra7178="EXTRA" -CHECK_SEVERITY_extra7178="High" -CHECK_ASFF_TYPE_extra7178="AwsEMR" -CHECK_ALTERNATE_check7178="extra7178" -CHECK_SERVICENAME_extra7178="emr" -CHECK_RISK_extra7178='EMR Clusters must have Account Public Access Block enabled' -CHECK_REMEDIATION_extra7178='Enable EMR Account Public Access Block' -CHECK_DOC_extra7178='https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html' -CHECK_CAF_EPIC_extra7178='Infrastructure Security' - -extra7178(){ - for regx in ${REGIONS}; do - block_public_access=$("${AWSCLI}" emr get-block-public-access-configuration ${PROFILE_OPT} --region "${regx}" --query 'BlockPublicAccessConfiguration.BlockPublicSecurityGroupRules' --output json 2>&1) - if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${block_public_access}"; then - textInfo "${regx}: Access Denied trying to get block public access configuration for EMR clusters" "${regx}" - continue - fi - if [[ "${block_public_access}" == "true" ]]; then - textPass "${regx}: EMR Account has Block Public Access enabled" "${regx}" - else - textFail "${regx}: EMR Account has Block Public Access disabled" "${regx}" - fi - done -} diff --git a/providers/aws/services/emr/emr_client.py b/providers/aws/services/emr/emr_client.py new file mode 100644 index 00000000..3e5084d8 --- /dev/null +++ b/providers/aws/services/emr/emr_client.py @@ -0,0 +1,4 @@ +from providers.aws.lib.audit_info.audit_info import current_audit_info +from providers.aws.services.emr.emr_service import EMR + +emr_client = EMR(current_audit_info) diff --git a/providers/aws/services/emr/emr_cluster_account_public_block_enabled/__init__.py b/providers/aws/services/emr/emr_cluster_account_public_block_enabled/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled.metadata.json b/providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled.metadata.json new file mode 100644 index 00000000..7fa53092 --- /dev/null +++ b/providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled.metadata.json @@ -0,0 +1,35 @@ +{ + "Provider": "aws", + "CheckID": "emr_cluster_account_public_block_enabled", + "CheckTitle": "EMR Account Public Access Block enabled.", + "CheckType": [], + "ServiceName": "emr", + "SubServiceName": "", + "ResourceIdTemplate": "arn:aws:emr:region:account-id", + "Severity": "high", + "ResourceType": "AwsEMR", + "Description": "EMR Account Public Access Block enabled.", + "Risk": "EMR Clusters must have Account Public Access Block enabled.", + "RelatedUrl": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html", + "Remediation": { + "Code": { + "CLI": "", + "NativeIaC": "", + "Other": "https://github.com/cloudmatos/matos/tree/master/remediations/aws/emr/block-emr-public-access", + "Terraform": "" + }, + "Recommendation": { + "Text": "Enable EMR Account Public Access Block.", + "Url": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html" + } + }, + "Categories": [], + "Tags": { + "Tag1Key": "value", + "Tag2Key": "value" + }, + "DependsOn": [], + "RelatedTo": [], + "Notes": "", + "Compliance": [] +} diff --git a/providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled.py b/providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled.py new file mode 100644 index 00000000..855c762e --- /dev/null +++ b/providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled.py @@ -0,0 +1,24 @@ +from lib.check.models import Check, Check_Report +from providers.aws.services.emr.emr_client import emr_client + + +class emr_cluster_account_public_block_enabled(Check): + def execute(self): + findings = [] + for region in emr_client.block_public_access_configuration: + report = Check_Report(self.metadata) + report.region = region + report.resource_id = emr_client.audited_account + + if emr_client.block_public_access_configuration[ + region + ].block_public_security_group_rules: + report.status = "PASS" + report.status_extended = "EMR Account has Block Public Access enabled" + else: + report.status = "FAIL" + report.status_extended = "EMR Account has Block Public Access disabled" + + findings.append(report) + + return findings diff --git a/providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled_test.py b/providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled_test.py new file mode 100644 index 00000000..5a77a6a7 --- /dev/null +++ b/providers/aws/services/emr/emr_cluster_account_public_block_enabled/emr_cluster_account_public_block_enabled_test.py @@ -0,0 +1,67 @@ +from unittest import mock + +from moto.core import DEFAULT_ACCOUNT_ID + +from providers.aws.services.emr.emr_service import BlockPublicAccessConfiguration + +AWS_REGION = "eu-west-1" + + +class Test_emr_cluster_account_public_block_enabled: + def test_account_public_block_enabled(self): + emr_client = mock.MagicMock + emr_client.audited_account = DEFAULT_ACCOUNT_ID + emr_client.block_public_access_configuration = { + AWS_REGION: BlockPublicAccessConfiguration( + block_public_security_group_rules=True + ) + } + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ): + # Test Check + from providers.aws.services.emr.emr_cluster_account_public_block_enabled.emr_cluster_account_public_block_enabled import ( + emr_cluster_account_public_block_enabled, + ) + + check = emr_cluster_account_public_block_enabled() + result = check.execute() + + assert len(result) == 1 + assert result[0].region == AWS_REGION + assert result[0].resource_id == DEFAULT_ACCOUNT_ID + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == "EMR Account has Block Public Access enabled" + ) + + def test_account_public_block_disabled(self): + emr_client = mock.MagicMock + emr_client.audited_account = DEFAULT_ACCOUNT_ID + emr_client.block_public_access_configuration = { + AWS_REGION: BlockPublicAccessConfiguration( + block_public_security_group_rules=False + ) + } + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ): + # Test Check + from providers.aws.services.emr.emr_cluster_account_public_block_enabled.emr_cluster_account_public_block_enabled import ( + emr_cluster_account_public_block_enabled, + ) + + check = emr_cluster_account_public_block_enabled() + result = check.execute() + + assert len(result) == 1 + assert result[0].region == AWS_REGION + assert result[0].resource_id == DEFAULT_ACCOUNT_ID + assert result[0].status == "FAIL" + assert ( + result[0].status_extended + == "EMR Account has Block Public Access disabled" + ) diff --git a/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/__init__.py b/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip.metadata.json b/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip.metadata.json new file mode 100644 index 00000000..4501e565 --- /dev/null +++ b/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip.metadata.json @@ -0,0 +1,35 @@ +{ + "Provider": "aws", + "CheckID": "emr_cluster_master_nodes_no_public_ip", + "CheckTitle": "EMR Cluster without Public IP.", + "CheckType": [], + "ServiceName": "emr", + "SubServiceName": "", + "ResourceIdTemplate": "arn:aws:emr:region:account-id", + "Severity": "medium", + "ResourceType": "AwsEMR", + "Description": "EMR Cluster without Public IP.", + "Risk": "EMR Cluster should not have Public IP.", + "RelatedUrl": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html", + "Remediation": { + "Code": { + "CLI": "", + "NativeIaC": "", + "Other": "", + "Terraform": "" + }, + "Recommendation": { + "Text": "Only make acceptable EMR clusters public.", + "Url": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html" + } + }, + "Categories": [], + "Tags": { + "Tag1Key": "value", + "Tag2Key": "value" + }, + "DependsOn": [], + "RelatedTo": [], + "Notes": "", + "Compliance": [] +} diff --git a/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip.py b/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip.py new file mode 100644 index 00000000..62a51aaa --- /dev/null +++ b/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip.py @@ -0,0 +1,30 @@ +from lib.check.models import Check, Check_Report +from providers.aws.services.emr.emr_client import emr_client +from providers.aws.services.emr.emr_service import ClusterStatus + + +class emr_cluster_master_nodes_no_public_ip(Check): + def execute(self): + findings = [] + for cluster in emr_client.clusters.values(): + if cluster.status not in ( + ClusterStatus.TERMINATED, + ClusterStatus.TERMINATED_WITH_ERRORS, + ): + report = Check_Report(self.metadata) + report.region = cluster.region + report.resource_id = cluster.id + report.resource_arn = cluster.arn + + if cluster.public: + report.status = "FAIL" + report.status_extended = f"EMR Cluster {cluster.id} has a Public IP" + else: + report.status = "PASS" + report.status_extended = ( + f"EMR Cluster {cluster.id} has not a Public IP" + ) + + findings.append(report) + + return findings diff --git a/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip_test.py b/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip_test.py new file mode 100644 index 00000000..c5bfad85 --- /dev/null +++ b/providers/aws/services/emr/emr_cluster_master_nodes_no_public_ip/emr_cluster_master_nodes_no_public_ip_test.py @@ -0,0 +1,169 @@ +from unittest import mock + +from moto.core import DEFAULT_ACCOUNT_ID + +from providers.aws.services.emr.emr_service import Cluster, ClusterStatus + +AWS_REGION = "eu-west-1" + + +class Test_emr_cluster_master_nodes_no_public_ip: + def test_no_clusters(self): + emr_client = mock.MagicMock + emr_client.clusters = {} + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ): + # Test Check + from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import ( + emr_cluster_master_nodes_no_public_ip, + ) + + check = emr_cluster_master_nodes_no_public_ip() + result = check.execute() + + assert len(result) == 0 + + def test_cluster_public_running(self): + emr_client = mock.MagicMock + cluster_name = "test-cluster" + cluster_id = "j-XWO1UKVCC6FCV" + cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}" + emr_client.clusters = { + "test-cluster": Cluster( + id=cluster_id, + arn=cluster_arn, + name=cluster_name, + status=ClusterStatus.RUNNING, + region=AWS_REGION, + master_public_dns_name="test.amazonaws.com", + public=True, + ) + } + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ): + # Test Check + from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import ( + emr_cluster_master_nodes_no_public_ip, + ) + + check = emr_cluster_master_nodes_no_public_ip() + result = check.execute() + + assert len(result) == 1 + assert result[0].region == AWS_REGION + assert result[0].resource_id == cluster_id + assert result[0].resource_arn == cluster_arn + assert result[0].status == "FAIL" + assert ( + result[0].status_extended == f"EMR Cluster {cluster_id} has a Public IP" + ) + + def test_cluster_private_running(self): + emr_client = mock.MagicMock + cluster_name = "test-cluster" + cluster_id = "j-XWO1UKVCC6FCV" + cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}" + emr_client.clusters = { + "test-cluster": Cluster( + id=cluster_id, + arn=cluster_arn, + name=cluster_name, + status=ClusterStatus.RUNNING, + region=AWS_REGION, + master_public_dns_name="compute.internal", + public=False, + ) + } + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ): + # Test Check + from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import ( + emr_cluster_master_nodes_no_public_ip, + ) + + check = emr_cluster_master_nodes_no_public_ip() + result = check.execute() + + assert len(result) == 1 + assert result[0].region == AWS_REGION + assert result[0].resource_id == cluster_id + assert result[0].resource_arn == cluster_arn + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == f"EMR Cluster {cluster_id} has not a Public IP" + ) + + def test_cluster_public_terminated(self): + emr_client = mock.MagicMock + cluster_name = "test-cluster" + cluster_id = "j-XWO1UKVCC6FCV" + cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}" + emr_client.clusters = { + "test-cluster": Cluster( + id=cluster_id, + arn=cluster_arn, + name=cluster_name, + status=ClusterStatus.TERMINATED, + region=AWS_REGION, + master_public_dns_name="test.amazonaws.com", + public=True, + ) + } + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ): + # Test Check + from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import ( + emr_cluster_master_nodes_no_public_ip, + ) + + check = emr_cluster_master_nodes_no_public_ip() + result = check.execute() + + assert len(result) == 0 + + def test_cluster_private_bootstrapping(self): + emr_client = mock.MagicMock + cluster_name = "test-cluster" + cluster_id = "j-XWO1UKVCC6FCV" + cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}" + emr_client.clusters = { + "test-cluster": Cluster( + id=cluster_id, + arn=cluster_arn, + name=cluster_name, + status=ClusterStatus.BOOTSTRAPPING, + region=AWS_REGION, + master_public_dns_name="compute.internal", + public=False, + ) + } + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ): + # Test Check + from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import ( + emr_cluster_master_nodes_no_public_ip, + ) + + check = emr_cluster_master_nodes_no_public_ip() + result = check.execute() + + assert len(result) == 1 + assert result[0].region == AWS_REGION + assert result[0].resource_id == cluster_id + assert result[0].resource_arn == cluster_arn + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == f"EMR Cluster {cluster_id} has not a Public IP" + ) diff --git a/providers/aws/services/emr/emr_cluster_publicly_accesible/__init__.py b/providers/aws/services/emr/emr_cluster_publicly_accesible/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible.metadata.json b/providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible.metadata.json new file mode 100644 index 00000000..c0171564 --- /dev/null +++ b/providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible.metadata.json @@ -0,0 +1,35 @@ +{ + "Provider": "aws", + "CheckID": "emr_cluster_publicly_accesible", + "CheckTitle": "Publicly accessible EMR Cluster.", + "CheckType": [], + "ServiceName": "emr", + "SubServiceName": "", + "ResourceIdTemplate": "arn:aws:emr:region:account-id", + "Severity": "medium", + "ResourceType": "AwsEMR", + "Description": "Publicly accessible EMR Cluster.", + "Risk": "EMR Clusters should not be publicly accessible.", + "RelatedUrl": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html", + "Remediation": { + "Code": { + "CLI": "", + "NativeIaC": "", + "Other": "", + "Terraform": "https://docs.bridgecrew.io/docs/ensure-that-amazon-emr-clusters-security-groups-are-not-open-to-the-world#terraform" + }, + "Recommendation": { + "Text": "Only make acceptable EMR clusters public.", + "Url": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html" + } + }, + "Categories": [], + "Tags": { + "Tag1Key": "value", + "Tag2Key": "value" + }, + "DependsOn": [], + "RelatedTo": [], + "Notes": "", + "Compliance": [] +} diff --git a/providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible.py b/providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible.py new file mode 100644 index 00000000..72b028f5 --- /dev/null +++ b/providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible.py @@ -0,0 +1,86 @@ +from copy import deepcopy + +from lib.check.models import Check, Check_Report +from providers.aws.services.ec2.ec2_client import ec2_client +from providers.aws.services.ec2.lib.security_groups import check_security_group +from providers.aws.services.emr.emr_client import emr_client +from providers.aws.services.emr.emr_service import ClusterStatus + + +class emr_cluster_publicly_accesible(Check): + def execute(self): + findings = [] + for cluster in emr_client.clusters.values(): + if cluster.status not in ( + ClusterStatus.TERMINATED, + ClusterStatus.TERMINATED_WITH_ERRORS, + ): + report = Check_Report(self.metadata) + report.region = cluster.region + report.resource_id = cluster.id + report.resource_arn = cluster.arn + + report.status = "PASS" + report.status_extended = ( + f"EMR Cluster {cluster.id} is not publicly accessible" + ) + # If EMR cluster is Public, it is required to check + # their Security Groups for the Master, + # the Slaves and the additional ones + if cluster.public: + + # Check Public Master Security Groups + master_node_sg_groups = deepcopy( + cluster.master.additional_security_groups_id + ) + master_node_sg_groups.append(cluster.master.security_group_id) + + master_public_security_groups = [] + for master_sg in master_node_sg_groups: + master_sg_public = False + for sg in ec2_client.security_groups: + if sg.id == master_sg: + for ingress_rule in sg.ingress_rules: + if check_security_group(ingress_rule, -1): + master_sg_public = True + break + if master_sg_public: + master_public_security_groups.append(sg.id) + break + + # Check Public Slave Security Groups + slave_node_sg_groups = deepcopy( + cluster.slave.additional_security_groups_id + ) + slave_node_sg_groups.append(cluster.slave.security_group_id) + + slave_public_security_groups = [] + for slave_sg in slave_node_sg_groups: + slave_sg_public = False + for sg in ec2_client.security_groups: + if sg.id == slave_sg: + for ingress_rule in sg.ingress_rules: + if check_security_group(ingress_rule, -1): + slave_sg_public = True + break + if slave_sg_public: + slave_public_security_groups.append(sg.id) + break + + if master_public_security_groups or slave_public_security_groups: + report.status = "FAIL" + report.status_extended = f"EMR Cluster {cluster.id} is publicly accessible through the following Security Groups:" + report.status_extended += ( + f" Master Node {master_public_security_groups}" + if master_public_security_groups + else "" + ) + report.status_extended += ( + f" Slaves Nodes {slave_public_security_groups}" + if slave_public_security_groups + else "" + ) + + findings.append(report) + + return findings diff --git a/providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible_test.py b/providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible_test.py new file mode 100644 index 00000000..ed623c51 --- /dev/null +++ b/providers/aws/services/emr/emr_cluster_publicly_accesible/emr_cluster_publicly_accesible_test.py @@ -0,0 +1,361 @@ +from unittest import mock +from uuid import uuid4 + +from boto3 import resource, session +from moto import mock_ec2 +from moto.core import DEFAULT_ACCOUNT_ID + +from providers.aws.lib.audit_info.audit_info import AWS_Audit_Info +from providers.aws.services.emr.emr_service import Cluster, ClusterStatus, Node + +AWS_REGION = "eu-west-1" + + +class Test_emr_cluster_publicly_accesible: + # Mocked Audit Info + def set_mocked_audit_info(self): + audit_info = AWS_Audit_Info( + original_session=None, + audit_session=session.Session( + profile_name=None, + botocore_session=None, + ), + audited_account=None, + audited_user_id=None, + audited_partition="aws", + audited_identity_arn=None, + profile=None, + profile_region=None, + credentials=None, + assumed_role_info=None, + audited_regions=None, + organizations_metadata=None, + ) + return audit_info + + def test_no_clusters(self): + # EMR Client + emr_client = mock.MagicMock + emr_client.clusters = {} + # EC2 Client + ec2_client = mock.MagicMock + + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ), mock.patch( + "providers.aws.services.ec2.ec2_service.EC2", + new=ec2_client, + ): + # Test Check + from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import ( + emr_cluster_publicly_accesible, + ) + + check = emr_cluster_publicly_accesible() + result = check.execute() + + assert len(result) == 0 + + @mock_ec2 + def test_clusters_master_public_sg(self): + # EC2 Client + ec2 = resource("ec2", AWS_REGION) + # Create Security Group + master_security_group = ec2.create_security_group( + GroupName=str(uuid4()), Description="test-decurity-group" + ) + master_security_group.authorize_ingress( + IpProtocol="tcp", + FromPort=0, + ToPort=65535, + CidrIp="0.0.0.0/0", + ) + + # EMR Client + emr_client = mock.MagicMock + cluster_name = "test-cluster" + cluster_id = "j-XWO1UKVCC6FCV" + cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}" + emr_client.clusters = { + "test-cluster": Cluster( + id=cluster_id, + arn=cluster_arn, + name=cluster_name, + status=ClusterStatus.RUNNING, + region=AWS_REGION, + master_public_dns_name="test.amazonaws.com", + public=True, + master=Node( + security_group_id=master_security_group.id, + additional_security_groups_id=[], + ), + ) + } + + master_expected_public_sgs = [master_security_group.id] + + from providers.aws.services.ec2.ec2_service import EC2 + + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ), mock.patch( + "providers.aws.lib.audit_info.audit_info.current_audit_info", + self.set_mocked_audit_info(), + ), mock.patch( + "providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client", + new=EC2(self.set_mocked_audit_info()), + ): + # Test Check + from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import ( + emr_cluster_publicly_accesible, + ) + + check = emr_cluster_publicly_accesible() + result = check.execute() + + assert len(result) == 1 + assert result[0].region == AWS_REGION + assert result[0].resource_id == cluster_id + assert result[0].resource_arn == cluster_arn + assert result[0].status == "FAIL" + assert ( + result[0].status_extended + == f"EMR Cluster {cluster_id} is publicly accessible through the following Security Groups: Master Node {master_expected_public_sgs}" + ) + + @mock_ec2 + def test_clusters_master_private_sg(self): + # EC2 Client + ec2 = resource("ec2", AWS_REGION) + # Create Security Group + master_security_group = ec2.create_security_group( + GroupName=str(uuid4()), Description="test-decurity-group" + ) + master_security_group.authorize_ingress( + IpProtocol="tcp", + FromPort=0, + ToPort=65535, + CidrIp="10.0.0.0/8", + ) + + # EMR Client + emr_client = mock.MagicMock + cluster_name = "test-cluster" + cluster_id = "j-XWO1UKVCC6FCV" + cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}" + emr_client.clusters = { + "test-cluster": Cluster( + id=cluster_id, + arn=cluster_arn, + name=cluster_name, + status=ClusterStatus.RUNNING, + region=AWS_REGION, + master_public_dns_name="test.amazonaws.com", + public=True, + master=Node( + security_group_id=master_security_group.id, + additional_security_groups_id=[], + ), + ) + } + + from providers.aws.services.ec2.ec2_service import EC2 + + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ), mock.patch( + "providers.aws.lib.audit_info.audit_info.current_audit_info", + self.set_mocked_audit_info(), + ), mock.patch( + "providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client", + new=EC2(self.set_mocked_audit_info()), + ): + # Test Check + from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import ( + emr_cluster_publicly_accesible, + ) + + check = emr_cluster_publicly_accesible() + result = check.execute() + + assert len(result) == 1 + assert result[0].region == AWS_REGION + assert result[0].resource_id == cluster_id + assert result[0].resource_arn == cluster_arn + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == f"EMR Cluster {cluster_id} is not publicly accessible" + ) + + @mock_ec2 + def test_clusters_master_private_slave_public_sg(self): + # EC2 Client + ec2 = resource("ec2", AWS_REGION) + # Create Master Security Group + master_security_group = ec2.create_security_group( + GroupName=str(uuid4()), Description="test-decurity-group" + ) + master_security_group.authorize_ingress( + IpProtocol="tcp", + FromPort=0, + ToPort=65535, + CidrIp="10.0.0.0/8", + ) + + # Create Slave Security Group + slave_security_group = ec2.create_security_group( + GroupName=str(uuid4()), Description="test-decurity-group" + ) + slave_security_group.authorize_ingress( + IpProtocol="tcp", + FromPort=0, + ToPort=65535, + CidrIp="0.0.0.0/0", + ) + + # EMR Client + emr_client = mock.MagicMock + cluster_name = "test-cluster" + cluster_id = "j-XWO1UKVCC6FCV" + cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}" + emr_client.clusters = { + "test-cluster": Cluster( + id=cluster_id, + arn=cluster_arn, + name=cluster_name, + status=ClusterStatus.RUNNING, + region=AWS_REGION, + master_public_dns_name="test.amazonaws.com", + public=True, + master=Node( + security_group_id=master_security_group.id, + additional_security_groups_id=[], + ), + slave=Node( + security_group_id=slave_security_group.id, + additional_security_groups_id=[], + ), + ) + } + + slave_expected_public_sgs = [slave_security_group.id] + + from providers.aws.services.ec2.ec2_service import EC2 + + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ), mock.patch( + "providers.aws.lib.audit_info.audit_info.current_audit_info", + self.set_mocked_audit_info(), + ), mock.patch( + "providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client", + new=EC2(self.set_mocked_audit_info()), + ): + # Test Check + from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import ( + emr_cluster_publicly_accesible, + ) + + check = emr_cluster_publicly_accesible() + result = check.execute() + + assert len(result) == 1 + assert result[0].region == AWS_REGION + assert result[0].resource_id == cluster_id + assert result[0].resource_arn == cluster_arn + assert result[0].status == "FAIL" + assert ( + result[0].status_extended + == f"EMR Cluster {cluster_id} is publicly accessible through the following Security Groups: Slaves Nodes {slave_expected_public_sgs}" + ) + + @mock_ec2 + def test_clusters_master_public_slave_private_two_sg(self): + # EC2 Client + ec2 = resource("ec2", AWS_REGION) + # Create Master Security Group + master_security_group = ec2.create_security_group( + GroupName=str(uuid4()), Description="test-decurity-group" + ) + master_security_group.authorize_ingress( + IpProtocol="tcp", + FromPort=0, + ToPort=65535, + CidrIp="0.0.0.0/0", + ) + + # Create Slave Security Group + slave_security_group = ec2.create_security_group( + GroupName=str(uuid4()), Description="test-decurity-group" + ) + slave_security_group.authorize_ingress( + IpProtocol="tcp", + FromPort=0, + ToPort=65535, + CidrIp="10.0.0.0/8", + ) + + # EMR Client + emr_client = mock.MagicMock + cluster_name = "test-cluster" + cluster_id = "j-XWO1UKVCC6FCV" + cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}" + emr_client.clusters = { + "test-cluster": Cluster( + id=cluster_id, + arn=cluster_arn, + name=cluster_name, + status=ClusterStatus.RUNNING, + region=AWS_REGION, + master_public_dns_name="test.amazonaws.com", + public=True, + master=Node( + security_group_id=master_security_group.id, + additional_security_groups_id=[master_security_group.id], + ), + slave=Node( + security_group_id=slave_security_group.id, + additional_security_groups_id=[slave_security_group.id], + ), + ) + } + + master_expected_public_sgs = [ + master_security_group.id, + master_security_group.id, + ] + + from providers.aws.services.ec2.ec2_service import EC2 + + with mock.patch( + "providers.aws.services.emr.emr_service.EMR", + new=emr_client, + ), mock.patch( + "providers.aws.lib.audit_info.audit_info.current_audit_info", + self.set_mocked_audit_info(), + ), mock.patch( + "providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client", + new=EC2(self.set_mocked_audit_info()), + ): + # Test Check + from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import ( + emr_cluster_publicly_accesible, + ) + + check = emr_cluster_publicly_accesible() + result = check.execute() + + assert len(result) == 1 + assert result[0].region == AWS_REGION + assert result[0].resource_id == cluster_id + assert result[0].resource_arn == cluster_arn + assert result[0].status == "FAIL" + assert ( + result[0].status_extended + == f"EMR Cluster {cluster_id} is publicly accessible through the following Security Groups: Master Node {master_expected_public_sgs}" + ) diff --git a/providers/aws/services/emr/emr_service.py b/providers/aws/services/emr/emr_service.py new file mode 100644 index 00000000..a92f42b3 --- /dev/null +++ b/providers/aws/services/emr/emr_service.py @@ -0,0 +1,166 @@ +import threading +from enum import Enum + +from pydantic import BaseModel + +from lib.logger import logger +from providers.aws.aws_provider import generate_regional_clients + + +################## EMR +class EMR: + def __init__(self, audit_info): + self.service = "emr" + self.session = audit_info.audit_session + self.audited_account = audit_info.audited_account + self.regional_clients = generate_regional_clients(self.service, audit_info) + self.clusters = {} + self.block_public_access_configuration = {} + self.__threading_call__(self.__list_clusters__) + self.__threading_call__(self.__describe_cluster__) + self.__threading_call__(self.__get_block_public_access_configuration__) + + def __get_session__(self): + return self.session + + def __threading_call__(self, call): + threads = [] + for regional_client in self.regional_clients.values(): + threads.append(threading.Thread(target=call, args=(regional_client,))) + for t in threads: + t.start() + for t in threads: + t.join() + + def __list_clusters__(self, regional_client): + logger.info("EMR - Listing Clusters...") + try: + list_clusters_paginator = regional_client.get_paginator("list_clusters") + for page in list_clusters_paginator.paginate(): + for cluster in page["Clusters"]: + cluster_name = cluster["Name"] + cluster_id = cluster["Id"] + cluster_arn = cluster["ClusterArn"] + cluster_status = cluster["Status"]["State"] + + self.clusters[cluster_id] = Cluster( + id=cluster_id, + name=cluster_name, + arn=cluster_arn, + status=cluster_status, + region=regional_client.region, + ) + + except Exception as error: + logger.error( + f"{regional_client.region} --" + f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:" + f" {error}" + ) + + def __describe_cluster__(self, regional_client): + logger.info("EMR - Describing Clusters...") + try: + for cluster in self.clusters.values(): + if cluster.region == regional_client.region: + describe_cluster_parameters = {"ClusterId": cluster.id} + cluster_info = regional_client.describe_cluster( + **describe_cluster_parameters + ) + + # Master Node Security Groups + master_node_security_group = cluster_info["Cluster"][ + "Ec2InstanceAttributes" + ]["EmrManagedMasterSecurityGroup"] + master_node_additional_security_groups = cluster_info["Cluster"][ + "Ec2InstanceAttributes" + ]["AdditionalMasterSecurityGroups"] + self.clusters[cluster.id].master = Node( + security_group_id=master_node_security_group, + additional_security_groups_id=master_node_additional_security_groups, + ) + + # Slave Node Security Groups + slave_node_security_group = cluster_info["Cluster"][ + "Ec2InstanceAttributes" + ]["EmrManagedSlaveSecurityGroup"] + slave_node_additional_security_groups = cluster_info["Cluster"][ + "Ec2InstanceAttributes" + ]["AdditionalSlaveSecurityGroups"] + self.clusters[cluster.id].slave = Node( + security_group_id=slave_node_security_group, + additional_security_groups_id=slave_node_additional_security_groups, + ) + + # Save MasterPublicDnsName + master_public_dns_name = cluster_info["Cluster"][ + "MasterPublicDnsName" + ] + self.clusters[ + cluster.id + ].master_public_dns_name = master_public_dns_name + # Set cluster Public/Private + # Public EMR cluster have their DNS ending with .amazonaws.com + # while private ones have format of ip-xxx-xx-xx.us-east-1.compute.internal. + if ".amazonaws.com" in master_public_dns_name: + self.clusters[cluster.id].public = True + + except Exception as error: + logger.error( + f"{regional_client.region} --" + f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:" + f" {error}" + ) + + def __get_block_public_access_configuration__(self, regional_client): + """Returns the Amazon EMR block public access configuration for your Amazon Web Services account in the current Region.""" + logger.info("EMR - Getting Block Public Access Configuration...") + try: + block_public_access_configuration = ( + regional_client.get_block_public_access_configuration() + ) + + self.block_public_access_configuration[ + regional_client.region + ] = BlockPublicAccessConfiguration( + block_public_security_group_rules=block_public_access_configuration[ + "BlockPublicAccessConfiguration" + ]["BlockPublicSecurityGroupRules"] + ) + except Exception as error: + logger.error( + f"{regional_client.region} --" + f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:" + f" {error}" + ) + + +class BlockPublicAccessConfiguration(BaseModel): + block_public_security_group_rules: bool + + +class ClusterStatus(Enum): + STARTING = "STARTING" + BOOTSTRAPPING = "BOOTSTRAPPING" + RUNNING = "RUNNING" + WAITING = "WAITING" + TERMINATING = "TERMINATING" + TERMINATED = "TERMINATED" + TERMINATED_WITH_ERRORS = "TERMINATED_WITH_ERRORS" + + +class Node(BaseModel): + security_group_id: str = "" + additional_security_groups_id: list[str] = [] + + +class Cluster(BaseModel): + id: str + name: str + status: ClusterStatus + arn: str + region: str + master: Node = Node() + slave: Node = Node() + master_public_dns_name: str = "" + public: bool = False diff --git a/providers/aws/services/emr/emr_service_test.py b/providers/aws/services/emr/emr_service_test.py new file mode 100644 index 00000000..f30b21d9 --- /dev/null +++ b/providers/aws/services/emr/emr_service_test.py @@ -0,0 +1,136 @@ +from datetime import datetime +from unittest.mock import patch + +import botocore +from boto3 import client, session +from moto import mock_emr +from moto.core import DEFAULT_ACCOUNT_ID + +from providers.aws.lib.audit_info.models import AWS_Audit_Info +from providers.aws.services.emr.emr_service import EMR, ClusterStatus + +# Mock Test Region +AWS_REGION = "eu-west-1" + + +# Mocking Access Analyzer Calls +make_api_call = botocore.client.BaseClient._make_api_call + + +def mock_make_api_call(self, operation_name, kwarg): + """We have to mock every AWS API call using Boto3""" + if operation_name == "GetBlockPublicAccessConfiguration": + return { + "BlockPublicAccessConfiguration": { + "BlockPublicSecurityGroupRules": True, + "PermittedPublicSecurityGroupRuleRanges": [ + {"MinRange": 0, "MaxRange": 65535}, + ], + }, + "BlockPublicAccessConfigurationMetadata": { + "CreationDateTime": datetime(2015, 1, 1), + "CreatedByArn": "test-arn", + }, + } + + return make_api_call(self, operation_name, kwarg) + + +# Mock generate_regional_clients() +def mock_generate_regional_clients(service, audit_info): + regional_client = audit_info.audit_session.client(service, region_name=AWS_REGION) + regional_client.region = AWS_REGION + return {AWS_REGION: regional_client} + + +@patch( + "providers.aws.services.emr.emr_service.generate_regional_clients", + new=mock_generate_regional_clients, +) +@patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call) +class Test_EMR_Service: + def set_mocked_audit_info(self): + audit_info = AWS_Audit_Info( + original_session=None, + audit_session=session.Session( + profile_name=None, + botocore_session=None, + ), + audited_account=DEFAULT_ACCOUNT_ID, + audited_user_id=None, + audited_partition="aws", + audited_identity_arn=None, + profile=None, + profile_region=None, + credentials=None, + assumed_role_info=None, + audited_regions=None, + organizations_metadata=None, + ) + return audit_info + + # Test EMR Client + @mock_emr + def test__get_client__(self): + emr = EMR(self.set_mocked_audit_info()) + assert emr.regional_clients[AWS_REGION].__class__.__name__ == "EMR" + + # Test EMR Session + @mock_emr + def test__get_session__(self): + emr = EMR(self.set_mocked_audit_info()) + assert emr.session.__class__.__name__ == "Session" + + # Test EMR Service + @mock_emr + def test__get_service__(self): + emr = EMR(self.set_mocked_audit_info()) + assert emr.service == "emr" + + # Test __list_clusters__ and __describe_cluster__ + @mock_emr + def test__list_clusters__(self): + # Create EMR Cluster + emr_client = client("emr", region_name=AWS_REGION) + cluster_name = "test-cluster" + run_job_flow_args = dict( + Instances={ + "InstanceCount": 3, + "KeepJobFlowAliveWhenNoSteps": True, + "MasterInstanceType": "c3.medium", + "Placement": {"AvailabilityZone": "us-east-1a"}, + "SlaveInstanceType": "c3.xlarge", + }, + JobFlowRole="EMR_EC2_DefaultRole", + LogUri="s3://mybucket/log", + Name=cluster_name, + ServiceRole="EMR_DefaultRole", + VisibleToAllUsers=True, + ) + cluster_id = emr_client.run_job_flow(**run_job_flow_args)["JobFlowId"] + # EMR Class + emr = EMR(self.set_mocked_audit_info()) + + assert len(emr.clusters) == 1 + assert emr.clusters[cluster_id].id == cluster_id + assert emr.clusters[cluster_id].name == cluster_name + assert emr.clusters[cluster_id].status == ClusterStatus.WAITING + assert ( + emr.clusters[cluster_id].arn + == f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_id}" + ) + assert emr.clusters[cluster_id].region == AWS_REGION + assert ( + emr.clusters[cluster_id].master_public_dns_name + == "ec2-184-0-0-1.us-west-1.compute.amazonaws.com" + ) + assert emr.clusters[cluster_id].public + + @mock_emr + def test__get_block_public_access_configuration__(self): + emr = EMR(self.set_mocked_audit_info()) + + assert len(emr.block_public_access_configuration) == 1 + assert emr.block_public_access_configuration[ + AWS_REGION + ].block_public_security_group_rules