feat(EMR): Service and checks (#1486)

Co-authored-by: Sergio Garcia <38561120+sergargar@users.noreply.github.com>
This commit is contained in:
Pepe Fagoaga
2022-11-17 11:31:20 +01:00
committed by GitHub
parent d2d2c75967
commit 24ca19d502
21 changed files with 1249 additions and 244 deletions

View File

@@ -1,27 +1,48 @@
import ipaddress
from typing import Any
################## Security Groups
# Check if the security group ingress rule has public access to the check_ports using the protocol
def check_security_group(ingress_rule: Any, protocol: str, ports: list = []) -> bool:
public_IPv4 = "0.0.0.0/0"
public_IPv6 = "::/0"
"""
Check if the security group ingress rule has public access to the check_ports using the protocol
@param ingress_rule: AWS Security Group IpPermissions Ingress Rule
{
'FromPort': 123,
'IpProtocol': 'string',
'IpRanges': [
{
'CidrIp': 'string',
'Description': 'string'
},
],
'Ipv6Ranges': [
{
'CidrIpv6': 'string',
'Description': 'string'
},
],
'ToPort': 123,
}
@param procotol: Protocol to check.
@param ports: List of ports to check. (Default: [])
"""
# Check for all traffic ingress rules regardless of the protocol
if ingress_rule["IpProtocol"] == "-1" and (
(
"0.0.0.0/0" in str(ingress_rule["IpRanges"])
or "::/0" in str(ingress_rule["Ipv6Ranges"])
)
):
return True
if ingress_rule["IpProtocol"] == "-1":
for ip_ingress_rule in ingress_rule["IpRanges"]:
if _is_cidr_public(ip_ingress_rule["CidrIp"]):
return True
for ip_ingress_rule in ingress_rule["Ipv6Ranges"]:
if _is_cidr_public(ip_ingress_rule["CidrIp"]):
return True
# Check for specific ports in ingress rules
if "FromPort" in ingress_rule:
# All ports
if ingress_rule["FromPort"] == 0 and ingress_rule["ToPort"] == 65535:
return True
# If there is a port range
if ingress_rule["FromPort"] != ingress_rule["ToPort"]:
# Calculate port range, adding 1
@@ -35,14 +56,49 @@ def check_security_group(ingress_rule: Any, protocol: str, ports: list = []) ->
ingress_port_range.append(int(ingress_rule["FromPort"]))
# Test Security Group
for port in ports:
if (
(
public_IPv4 in str(ingress_rule["IpRanges"])
or public_IPv6 in str(ingress_rule["Ipv6Ranges"])
)
and port in ingress_port_range
and ingress_rule["IpProtocol"] == protocol
):
return True
# IPv4
for ip_ingress_rule in ingress_rule["IpRanges"]:
if _is_cidr_public(ip_ingress_rule["CidrIp"]):
# If there are input ports to check
if ports:
for port in ports:
if (
port in ingress_port_range
and ingress_rule["IpProtocol"] == protocol
):
return True
else:
return True
# IPv6
for ip_ingress_rule in ingress_rule["Ipv6Ranges"]:
if _is_cidr_public(ip_ingress_rule["CidrIp"]):
# If there are input ports to check
if ports:
for port in ports:
if (
port in ingress_port_range
and ingress_rule["IpProtocol"] == protocol
):
return True
else:
return True
return False
def _is_cidr_public(cidr: str) -> bool:
"""
Check if an input CIDR is public
@param cidr: CIDR 10.22.33.44/8
"""
public_IPv4 = "0.0.0.0/0"
public_IPv6 = "::/0"
# Workaround until this issue is fixed
# PR https://github.com/python/cpython/pull/97733
# Issue https://github.com/python/cpython/issues/82836
if cidr in (public_IPv4, public_IPv6):
return True
return ipaddress.ip_network(cidr).is_global

View File

@@ -0,0 +1,21 @@
import pytest
from providers.aws.services.ec2.lib.security_groups import _is_cidr_public
class Test_security_groups:
def test__is_cidr_public_Public_IP(self):
cidr = "0.0.0.0/0"
assert _is_cidr_public(cidr)
def test__is_cidr_public_Private_IP(self):
cidr = "10.0.0.0/8"
assert not _is_cidr_public(cidr)
def test__is_cidr_public_Bad_Private_IP(self):
cidr = "10.0.0.0/0"
with pytest.raises(ValueError) as ex:
_is_cidr_public(cidr)
assert ex.type == ValueError
assert ex.match(f"{cidr} has host bits set")

View File

View File

@@ -1,55 +0,0 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
CHECK_ID_extra7176="7.176"
CHECK_TITLE_extra7176="[extra7176] EMR Cluster without Public IP"
CHECK_SCORED_extra7176="NOT_SCORED"
CHECK_CIS_LEVEL_extra7176="EXTRA"
CHECK_SEVERITY_extra7176="Medium"
CHECK_ASFF_TYPE_extra7176="AwsEMR"
CHECK_ALTERNATE_check7176="extra7176"
CHECK_SERVICENAME_extra7176="emr"
CHECK_RISK_extra7176='EMR Cluster should not have Public IP'
CHECK_REMEDIATION_extra7176='Only make acceptable EMR clusters public'
CHECK_DOC_extra7176='https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html'
CHECK_CAF_EPIC_extra7176='Infrastructure Security'
extra7176(){
# Public EMR cluster have their DNS ending with .amazonaws.com while private ones have format of ip-xxx-xx-xx.us-east-1.compute.internal.
for regx in ${REGIONS}; do
# List only EMR clusters with the following states: STARTING, BOOTSTRAPPING, RUNNING, WAITING, TERMINATING
# [NOT TERMINATED AND TERMINATED_WITH_ERRORS]
LIST_OF_CLUSTERS=$("${AWSCLI}" emr list-clusters ${PROFILE_OPT} --region "${regx}" --query 'Clusters[?(Status.State!=`TERMINATED` && Status.State!=`TERMINATED_WITH_ERRORS`)].Id' --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${LIST_OF_CLUSTERS}"; then
textInfo "${regx}: Access Denied trying to list clusters of emr" "${regx}"
continue
fi
if [[ "${LIST_OF_CLUSTERS}" ]]
then
for cluster_id in ${LIST_OF_CLUSTERS}; do
master_public_dns=$("${AWSCLI}" emr describe-cluster ${PROFILE_OPT} --cluster-id "${cluster_id}" --query 'Cluster.MasterPublicDnsName' --region "${regx}" --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_public_dns}"; then
textInfo "${regx}: Access Denied trying to describe emr cluster" "${regx}" "${cluster_id}"
continue
fi
if [[ $master_public_dns != None && $master_public_dns != *.internal ]];then
textFail "${regx}: EMR Cluster ${cluster_id} has a Public IP" "${regx}" "${cluster_id}"
else
textPass "${regx}: EMR Cluster ${cluster_id} has not a Public IP" "${regx}" "${cluster_id}"
fi
done
else
textInfo "${regx}: No EMR Clusters found" "${regx}"
fi
done
}

View File

@@ -1,124 +0,0 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
CHECK_ID_extra7177="7.177"
CHECK_TITLE_extra7177="[extra7177] Publicly accessible EMR Cluster"
CHECK_SCORED_extra7177="NOT_SCORED"
CHECK_CIS_LEVEL_extra7177="EXTRA"
CHECK_SEVERITY_extra7177="High"
CHECK_ASFF_TYPE_extra7177="AwsEMR"
CHECK_ALTERNATE_check7177="extra7177"
CHECK_SERVICENAME_extra7177="emr"
CHECK_RISK_extra7177='EMR Clusters should not be publicly accessible'
CHECK_REMEDIATION_extra7177='Only make acceptable EMR clusters public'
CHECK_DOC_extra7177='https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html'
CHECK_CAF_EPIC_extra7177='Infrastructure Security'
extra7177(){
for regx in ${REGIONS}; do
# List only EMR clusters with the following states: STARTING, BOOTSTRAPPING, RUNNING, WAITING, TERMINATING
# [NOT TERMINATED AND TERMINATED_WITH_ERRORS]
LIST_OF_CLUSTERS=$("${AWSCLI}" emr list-clusters ${PROFILE_OPT} --region "${regx}" --query 'Clusters[?(Status.State!=`TERMINATED` && Status.State!=`TERMINATED_WITH_ERRORS`)].Id' --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${LIST_OF_CLUSTERS}"; then
textInfo "${regx}: Access Denied trying to list EMR clusters" "${regx}"
continue
fi
if [[ "${LIST_OF_CLUSTERS}" ]]
then
for cluster_id in ${LIST_OF_CLUSTERS}; do
master_public_dns=$("${AWSCLI}" emr describe-cluster ${PROFILE_OPT} --cluster-id "${cluster_id}" --query 'Cluster.MasterPublicDnsName' --region "${regx}" --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_public_dns}"; then
textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}"
continue
fi
if [[ $master_public_dns != None && $master_public_dns != *.internal ]];then
# If EMR cluster is Public, it is required to check their Security Groups for the Master, the Slaves and the additional ones
# Retrive EMR Master Node Security Groups rules
master_node_sg=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.EmrManagedMasterSecurityGroup' --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_node_sg}"; then
textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}"
continue
fi
master_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${master_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_node_sg_internet_open}"; then
textInfo "$regx: Access Denied trying to describe security groups" "$regx"
continue
fi
# Retrive EMR Slave Node Security Groups rules
slave_node_sg=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.EmrManagedSlaveSecurityGroup' --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg}"; then
textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}"
continue
fi
slave_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${slave_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg_internet_open}"; then
textInfo "$regx: Access Denied trying to describe security groups" "$regx"
continue
fi
# Retrive EMR Additional Master node Security Groups rules
additional_master_node_sg_list=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.AdditionalMasterSecurityGroups' --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg}"; then
textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}"
continue
fi
local additional_master_node_sg_internet_open_list
if [[ "${additional_master_node_sg_list}" != "None" ]]; then
for additional_master_node_sg in ${additional_master_node_sg_list}; do
additional_master_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${additional_master_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg_internet_open}"; then
textInfo "$regx: Access Denied trying to describe security groups" "$regx"
continue
fi
# Store additional master node security groups that allows access from the internet
additional_master_node_sg_internet_open_list+=( "${additional_master_node_sg_internet_open}" )
done
fi
# Retrive EMR Additional Slave node Security Groups rules
additional_slave_node_sg_list=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.AdditionalSlaveSecurityGroups' --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg}"; then
textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}"
continue
fi
local additional_slave_node_sg_internet_open_list
if [[ "${additional_slave_node_sg_list}" != "None" ]]; then
for additional_slave_node_sg in ${additional_master_node_sg_list}; do
additional_slave_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${additional_slave_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${additional_slave_node_sg_internet_open}"; then
textInfo "$regx: Access Denied trying to describe security groups" "$regx"
continue
fi
# Store additional slave node security groups that allows access from the internet
additional_slave_node_sg_internet_open_list+=( "${additional_slave_node_sg_internet_open}" )
done
fi
# Check if EMR Cluster is publicly accessible through a Security Group
if [[ -n "${master_node_sg_internet_open}" || -n "${slave_node_sg_internet_open}" || "${#additional_master_node_sg_internet_open_list[@]}" -ne 0 || "${#additional_slave_node_sg_internet_open_list[@]}" -ne 0 ]]; then
textFail "${regx}: EMR Cluster ${cluster_id} is publicly accessible through the following Security Groups: Master Node ${master_node_sg_internet_open} ${additional_master_node_sg_internet_open_list[*]} -- Slaves Nodes ${slave_node_sg_internet_open} ${additional_slave_node_sg_internet_open_list[*]}" "${regx}" "${cluster_id}"
else
textPass "${regx}: EMR Cluster ${cluster_id} is not publicly accessible" "${regx}" "${cluster_id}"
fi
else
textPass "${regx}: EMR Cluster ${cluster_id} is not publicly accessible" "${regx}" "${cluster_id}"
fi
done
else
textInfo "${regx}: No EMR Clusters found" "${regx}"
fi
done
}

View File

@@ -1,41 +0,0 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
CHECK_ID_extra7178="7.178"
CHECK_TITLE_extra7178="[extra7178] EMR Account Public Access Block enabled"
CHECK_SCORED_extra7178="NOT_SCORED"
CHECK_CIS_LEVEL_extra7178="EXTRA"
CHECK_SEVERITY_extra7178="High"
CHECK_ASFF_TYPE_extra7178="AwsEMR"
CHECK_ALTERNATE_check7178="extra7178"
CHECK_SERVICENAME_extra7178="emr"
CHECK_RISK_extra7178='EMR Clusters must have Account Public Access Block enabled'
CHECK_REMEDIATION_extra7178='Enable EMR Account Public Access Block'
CHECK_DOC_extra7178='https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html'
CHECK_CAF_EPIC_extra7178='Infrastructure Security'
extra7178(){
for regx in ${REGIONS}; do
block_public_access=$("${AWSCLI}" emr get-block-public-access-configuration ${PROFILE_OPT} --region "${regx}" --query 'BlockPublicAccessConfiguration.BlockPublicSecurityGroupRules' --output json 2>&1)
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${block_public_access}"; then
textInfo "${regx}: Access Denied trying to get block public access configuration for EMR clusters" "${regx}"
continue
fi
if [[ "${block_public_access}" == "true" ]]; then
textPass "${regx}: EMR Account has Block Public Access enabled" "${regx}"
else
textFail "${regx}: EMR Account has Block Public Access disabled" "${regx}"
fi
done
}

View File

@@ -0,0 +1,4 @@
from providers.aws.lib.audit_info.audit_info import current_audit_info
from providers.aws.services.emr.emr_service import EMR
emr_client = EMR(current_audit_info)

View File

@@ -0,0 +1,35 @@
{
"Provider": "aws",
"CheckID": "emr_cluster_account_public_block_enabled",
"CheckTitle": "EMR Account Public Access Block enabled.",
"CheckType": [],
"ServiceName": "emr",
"SubServiceName": "",
"ResourceIdTemplate": "arn:aws:emr:region:account-id",
"Severity": "high",
"ResourceType": "AwsEMR",
"Description": "EMR Account Public Access Block enabled.",
"Risk": "EMR Clusters must have Account Public Access Block enabled.",
"RelatedUrl": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html",
"Remediation": {
"Code": {
"CLI": "",
"NativeIaC": "",
"Other": "https://github.com/cloudmatos/matos/tree/master/remediations/aws/emr/block-emr-public-access",
"Terraform": ""
},
"Recommendation": {
"Text": "Enable EMR Account Public Access Block.",
"Url": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html"
}
},
"Categories": [],
"Tags": {
"Tag1Key": "value",
"Tag2Key": "value"
},
"DependsOn": [],
"RelatedTo": [],
"Notes": "",
"Compliance": []
}

View File

@@ -0,0 +1,24 @@
from lib.check.models import Check, Check_Report
from providers.aws.services.emr.emr_client import emr_client
class emr_cluster_account_public_block_enabled(Check):
def execute(self):
findings = []
for region in emr_client.block_public_access_configuration:
report = Check_Report(self.metadata)
report.region = region
report.resource_id = emr_client.audited_account
if emr_client.block_public_access_configuration[
region
].block_public_security_group_rules:
report.status = "PASS"
report.status_extended = "EMR Account has Block Public Access enabled"
else:
report.status = "FAIL"
report.status_extended = "EMR Account has Block Public Access disabled"
findings.append(report)
return findings

View File

@@ -0,0 +1,67 @@
from unittest import mock
from moto.core import DEFAULT_ACCOUNT_ID
from providers.aws.services.emr.emr_service import BlockPublicAccessConfiguration
AWS_REGION = "eu-west-1"
class Test_emr_cluster_account_public_block_enabled:
def test_account_public_block_enabled(self):
emr_client = mock.MagicMock
emr_client.audited_account = DEFAULT_ACCOUNT_ID
emr_client.block_public_access_configuration = {
AWS_REGION: BlockPublicAccessConfiguration(
block_public_security_group_rules=True
)
}
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
):
# Test Check
from providers.aws.services.emr.emr_cluster_account_public_block_enabled.emr_cluster_account_public_block_enabled import (
emr_cluster_account_public_block_enabled,
)
check = emr_cluster_account_public_block_enabled()
result = check.execute()
assert len(result) == 1
assert result[0].region == AWS_REGION
assert result[0].resource_id == DEFAULT_ACCOUNT_ID
assert result[0].status == "PASS"
assert (
result[0].status_extended
== "EMR Account has Block Public Access enabled"
)
def test_account_public_block_disabled(self):
emr_client = mock.MagicMock
emr_client.audited_account = DEFAULT_ACCOUNT_ID
emr_client.block_public_access_configuration = {
AWS_REGION: BlockPublicAccessConfiguration(
block_public_security_group_rules=False
)
}
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
):
# Test Check
from providers.aws.services.emr.emr_cluster_account_public_block_enabled.emr_cluster_account_public_block_enabled import (
emr_cluster_account_public_block_enabled,
)
check = emr_cluster_account_public_block_enabled()
result = check.execute()
assert len(result) == 1
assert result[0].region == AWS_REGION
assert result[0].resource_id == DEFAULT_ACCOUNT_ID
assert result[0].status == "FAIL"
assert (
result[0].status_extended
== "EMR Account has Block Public Access disabled"
)

View File

@@ -0,0 +1,35 @@
{
"Provider": "aws",
"CheckID": "emr_cluster_master_nodes_no_public_ip",
"CheckTitle": "EMR Cluster without Public IP.",
"CheckType": [],
"ServiceName": "emr",
"SubServiceName": "",
"ResourceIdTemplate": "arn:aws:emr:region:account-id",
"Severity": "medium",
"ResourceType": "AwsEMR",
"Description": "EMR Cluster without Public IP.",
"Risk": "EMR Cluster should not have Public IP.",
"RelatedUrl": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html",
"Remediation": {
"Code": {
"CLI": "",
"NativeIaC": "",
"Other": "",
"Terraform": ""
},
"Recommendation": {
"Text": "Only make acceptable EMR clusters public.",
"Url": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html"
}
},
"Categories": [],
"Tags": {
"Tag1Key": "value",
"Tag2Key": "value"
},
"DependsOn": [],
"RelatedTo": [],
"Notes": "",
"Compliance": []
}

View File

@@ -0,0 +1,30 @@
from lib.check.models import Check, Check_Report
from providers.aws.services.emr.emr_client import emr_client
from providers.aws.services.emr.emr_service import ClusterStatus
class emr_cluster_master_nodes_no_public_ip(Check):
def execute(self):
findings = []
for cluster in emr_client.clusters.values():
if cluster.status not in (
ClusterStatus.TERMINATED,
ClusterStatus.TERMINATED_WITH_ERRORS,
):
report = Check_Report(self.metadata)
report.region = cluster.region
report.resource_id = cluster.id
report.resource_arn = cluster.arn
if cluster.public:
report.status = "FAIL"
report.status_extended = f"EMR Cluster {cluster.id} has a Public IP"
else:
report.status = "PASS"
report.status_extended = (
f"EMR Cluster {cluster.id} has not a Public IP"
)
findings.append(report)
return findings

View File

@@ -0,0 +1,169 @@
from unittest import mock
from moto.core import DEFAULT_ACCOUNT_ID
from providers.aws.services.emr.emr_service import Cluster, ClusterStatus
AWS_REGION = "eu-west-1"
class Test_emr_cluster_master_nodes_no_public_ip:
def test_no_clusters(self):
emr_client = mock.MagicMock
emr_client.clusters = {}
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
):
# Test Check
from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import (
emr_cluster_master_nodes_no_public_ip,
)
check = emr_cluster_master_nodes_no_public_ip()
result = check.execute()
assert len(result) == 0
def test_cluster_public_running(self):
emr_client = mock.MagicMock
cluster_name = "test-cluster"
cluster_id = "j-XWO1UKVCC6FCV"
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
emr_client.clusters = {
"test-cluster": Cluster(
id=cluster_id,
arn=cluster_arn,
name=cluster_name,
status=ClusterStatus.RUNNING,
region=AWS_REGION,
master_public_dns_name="test.amazonaws.com",
public=True,
)
}
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
):
# Test Check
from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import (
emr_cluster_master_nodes_no_public_ip,
)
check = emr_cluster_master_nodes_no_public_ip()
result = check.execute()
assert len(result) == 1
assert result[0].region == AWS_REGION
assert result[0].resource_id == cluster_id
assert result[0].resource_arn == cluster_arn
assert result[0].status == "FAIL"
assert (
result[0].status_extended == f"EMR Cluster {cluster_id} has a Public IP"
)
def test_cluster_private_running(self):
emr_client = mock.MagicMock
cluster_name = "test-cluster"
cluster_id = "j-XWO1UKVCC6FCV"
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
emr_client.clusters = {
"test-cluster": Cluster(
id=cluster_id,
arn=cluster_arn,
name=cluster_name,
status=ClusterStatus.RUNNING,
region=AWS_REGION,
master_public_dns_name="compute.internal",
public=False,
)
}
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
):
# Test Check
from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import (
emr_cluster_master_nodes_no_public_ip,
)
check = emr_cluster_master_nodes_no_public_ip()
result = check.execute()
assert len(result) == 1
assert result[0].region == AWS_REGION
assert result[0].resource_id == cluster_id
assert result[0].resource_arn == cluster_arn
assert result[0].status == "PASS"
assert (
result[0].status_extended
== f"EMR Cluster {cluster_id} has not a Public IP"
)
def test_cluster_public_terminated(self):
emr_client = mock.MagicMock
cluster_name = "test-cluster"
cluster_id = "j-XWO1UKVCC6FCV"
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
emr_client.clusters = {
"test-cluster": Cluster(
id=cluster_id,
arn=cluster_arn,
name=cluster_name,
status=ClusterStatus.TERMINATED,
region=AWS_REGION,
master_public_dns_name="test.amazonaws.com",
public=True,
)
}
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
):
# Test Check
from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import (
emr_cluster_master_nodes_no_public_ip,
)
check = emr_cluster_master_nodes_no_public_ip()
result = check.execute()
assert len(result) == 0
def test_cluster_private_bootstrapping(self):
emr_client = mock.MagicMock
cluster_name = "test-cluster"
cluster_id = "j-XWO1UKVCC6FCV"
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
emr_client.clusters = {
"test-cluster": Cluster(
id=cluster_id,
arn=cluster_arn,
name=cluster_name,
status=ClusterStatus.BOOTSTRAPPING,
region=AWS_REGION,
master_public_dns_name="compute.internal",
public=False,
)
}
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
):
# Test Check
from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import (
emr_cluster_master_nodes_no_public_ip,
)
check = emr_cluster_master_nodes_no_public_ip()
result = check.execute()
assert len(result) == 1
assert result[0].region == AWS_REGION
assert result[0].resource_id == cluster_id
assert result[0].resource_arn == cluster_arn
assert result[0].status == "PASS"
assert (
result[0].status_extended
== f"EMR Cluster {cluster_id} has not a Public IP"
)

View File

@@ -0,0 +1,35 @@
{
"Provider": "aws",
"CheckID": "emr_cluster_publicly_accesible",
"CheckTitle": "Publicly accessible EMR Cluster.",
"CheckType": [],
"ServiceName": "emr",
"SubServiceName": "",
"ResourceIdTemplate": "arn:aws:emr:region:account-id",
"Severity": "medium",
"ResourceType": "AwsEMR",
"Description": "Publicly accessible EMR Cluster.",
"Risk": "EMR Clusters should not be publicly accessible.",
"RelatedUrl": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html",
"Remediation": {
"Code": {
"CLI": "",
"NativeIaC": "",
"Other": "",
"Terraform": "https://docs.bridgecrew.io/docs/ensure-that-amazon-emr-clusters-security-groups-are-not-open-to-the-world#terraform"
},
"Recommendation": {
"Text": "Only make acceptable EMR clusters public.",
"Url": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html"
}
},
"Categories": [],
"Tags": {
"Tag1Key": "value",
"Tag2Key": "value"
},
"DependsOn": [],
"RelatedTo": [],
"Notes": "",
"Compliance": []
}

View File

@@ -0,0 +1,86 @@
from copy import deepcopy
from lib.check.models import Check, Check_Report
from providers.aws.services.ec2.ec2_client import ec2_client
from providers.aws.services.ec2.lib.security_groups import check_security_group
from providers.aws.services.emr.emr_client import emr_client
from providers.aws.services.emr.emr_service import ClusterStatus
class emr_cluster_publicly_accesible(Check):
def execute(self):
findings = []
for cluster in emr_client.clusters.values():
if cluster.status not in (
ClusterStatus.TERMINATED,
ClusterStatus.TERMINATED_WITH_ERRORS,
):
report = Check_Report(self.metadata)
report.region = cluster.region
report.resource_id = cluster.id
report.resource_arn = cluster.arn
report.status = "PASS"
report.status_extended = (
f"EMR Cluster {cluster.id} is not publicly accessible"
)
# If EMR cluster is Public, it is required to check
# their Security Groups for the Master,
# the Slaves and the additional ones
if cluster.public:
# Check Public Master Security Groups
master_node_sg_groups = deepcopy(
cluster.master.additional_security_groups_id
)
master_node_sg_groups.append(cluster.master.security_group_id)
master_public_security_groups = []
for master_sg in master_node_sg_groups:
master_sg_public = False
for sg in ec2_client.security_groups:
if sg.id == master_sg:
for ingress_rule in sg.ingress_rules:
if check_security_group(ingress_rule, -1):
master_sg_public = True
break
if master_sg_public:
master_public_security_groups.append(sg.id)
break
# Check Public Slave Security Groups
slave_node_sg_groups = deepcopy(
cluster.slave.additional_security_groups_id
)
slave_node_sg_groups.append(cluster.slave.security_group_id)
slave_public_security_groups = []
for slave_sg in slave_node_sg_groups:
slave_sg_public = False
for sg in ec2_client.security_groups:
if sg.id == slave_sg:
for ingress_rule in sg.ingress_rules:
if check_security_group(ingress_rule, -1):
slave_sg_public = True
break
if slave_sg_public:
slave_public_security_groups.append(sg.id)
break
if master_public_security_groups or slave_public_security_groups:
report.status = "FAIL"
report.status_extended = f"EMR Cluster {cluster.id} is publicly accessible through the following Security Groups:"
report.status_extended += (
f" Master Node {master_public_security_groups}"
if master_public_security_groups
else ""
)
report.status_extended += (
f" Slaves Nodes {slave_public_security_groups}"
if slave_public_security_groups
else ""
)
findings.append(report)
return findings

View File

@@ -0,0 +1,361 @@
from unittest import mock
from uuid import uuid4
from boto3 import resource, session
from moto import mock_ec2
from moto.core import DEFAULT_ACCOUNT_ID
from providers.aws.lib.audit_info.audit_info import AWS_Audit_Info
from providers.aws.services.emr.emr_service import Cluster, ClusterStatus, Node
AWS_REGION = "eu-west-1"
class Test_emr_cluster_publicly_accesible:
# Mocked Audit Info
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
original_session=None,
audit_session=session.Session(
profile_name=None,
botocore_session=None,
),
audited_account=None,
audited_user_id=None,
audited_partition="aws",
audited_identity_arn=None,
profile=None,
profile_region=None,
credentials=None,
assumed_role_info=None,
audited_regions=None,
organizations_metadata=None,
)
return audit_info
def test_no_clusters(self):
# EMR Client
emr_client = mock.MagicMock
emr_client.clusters = {}
# EC2 Client
ec2_client = mock.MagicMock
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
), mock.patch(
"providers.aws.services.ec2.ec2_service.EC2",
new=ec2_client,
):
# Test Check
from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import (
emr_cluster_publicly_accesible,
)
check = emr_cluster_publicly_accesible()
result = check.execute()
assert len(result) == 0
@mock_ec2
def test_clusters_master_public_sg(self):
# EC2 Client
ec2 = resource("ec2", AWS_REGION)
# Create Security Group
master_security_group = ec2.create_security_group(
GroupName=str(uuid4()), Description="test-decurity-group"
)
master_security_group.authorize_ingress(
IpProtocol="tcp",
FromPort=0,
ToPort=65535,
CidrIp="0.0.0.0/0",
)
# EMR Client
emr_client = mock.MagicMock
cluster_name = "test-cluster"
cluster_id = "j-XWO1UKVCC6FCV"
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
emr_client.clusters = {
"test-cluster": Cluster(
id=cluster_id,
arn=cluster_arn,
name=cluster_name,
status=ClusterStatus.RUNNING,
region=AWS_REGION,
master_public_dns_name="test.amazonaws.com",
public=True,
master=Node(
security_group_id=master_security_group.id,
additional_security_groups_id=[],
),
)
}
master_expected_public_sgs = [master_security_group.id]
from providers.aws.services.ec2.ec2_service import EC2
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
), mock.patch(
"providers.aws.lib.audit_info.audit_info.current_audit_info",
self.set_mocked_audit_info(),
), mock.patch(
"providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client",
new=EC2(self.set_mocked_audit_info()),
):
# Test Check
from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import (
emr_cluster_publicly_accesible,
)
check = emr_cluster_publicly_accesible()
result = check.execute()
assert len(result) == 1
assert result[0].region == AWS_REGION
assert result[0].resource_id == cluster_id
assert result[0].resource_arn == cluster_arn
assert result[0].status == "FAIL"
assert (
result[0].status_extended
== f"EMR Cluster {cluster_id} is publicly accessible through the following Security Groups: Master Node {master_expected_public_sgs}"
)
@mock_ec2
def test_clusters_master_private_sg(self):
# EC2 Client
ec2 = resource("ec2", AWS_REGION)
# Create Security Group
master_security_group = ec2.create_security_group(
GroupName=str(uuid4()), Description="test-decurity-group"
)
master_security_group.authorize_ingress(
IpProtocol="tcp",
FromPort=0,
ToPort=65535,
CidrIp="10.0.0.0/8",
)
# EMR Client
emr_client = mock.MagicMock
cluster_name = "test-cluster"
cluster_id = "j-XWO1UKVCC6FCV"
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
emr_client.clusters = {
"test-cluster": Cluster(
id=cluster_id,
arn=cluster_arn,
name=cluster_name,
status=ClusterStatus.RUNNING,
region=AWS_REGION,
master_public_dns_name="test.amazonaws.com",
public=True,
master=Node(
security_group_id=master_security_group.id,
additional_security_groups_id=[],
),
)
}
from providers.aws.services.ec2.ec2_service import EC2
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
), mock.patch(
"providers.aws.lib.audit_info.audit_info.current_audit_info",
self.set_mocked_audit_info(),
), mock.patch(
"providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client",
new=EC2(self.set_mocked_audit_info()),
):
# Test Check
from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import (
emr_cluster_publicly_accesible,
)
check = emr_cluster_publicly_accesible()
result = check.execute()
assert len(result) == 1
assert result[0].region == AWS_REGION
assert result[0].resource_id == cluster_id
assert result[0].resource_arn == cluster_arn
assert result[0].status == "PASS"
assert (
result[0].status_extended
== f"EMR Cluster {cluster_id} is not publicly accessible"
)
@mock_ec2
def test_clusters_master_private_slave_public_sg(self):
# EC2 Client
ec2 = resource("ec2", AWS_REGION)
# Create Master Security Group
master_security_group = ec2.create_security_group(
GroupName=str(uuid4()), Description="test-decurity-group"
)
master_security_group.authorize_ingress(
IpProtocol="tcp",
FromPort=0,
ToPort=65535,
CidrIp="10.0.0.0/8",
)
# Create Slave Security Group
slave_security_group = ec2.create_security_group(
GroupName=str(uuid4()), Description="test-decurity-group"
)
slave_security_group.authorize_ingress(
IpProtocol="tcp",
FromPort=0,
ToPort=65535,
CidrIp="0.0.0.0/0",
)
# EMR Client
emr_client = mock.MagicMock
cluster_name = "test-cluster"
cluster_id = "j-XWO1UKVCC6FCV"
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
emr_client.clusters = {
"test-cluster": Cluster(
id=cluster_id,
arn=cluster_arn,
name=cluster_name,
status=ClusterStatus.RUNNING,
region=AWS_REGION,
master_public_dns_name="test.amazonaws.com",
public=True,
master=Node(
security_group_id=master_security_group.id,
additional_security_groups_id=[],
),
slave=Node(
security_group_id=slave_security_group.id,
additional_security_groups_id=[],
),
)
}
slave_expected_public_sgs = [slave_security_group.id]
from providers.aws.services.ec2.ec2_service import EC2
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
), mock.patch(
"providers.aws.lib.audit_info.audit_info.current_audit_info",
self.set_mocked_audit_info(),
), mock.patch(
"providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client",
new=EC2(self.set_mocked_audit_info()),
):
# Test Check
from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import (
emr_cluster_publicly_accesible,
)
check = emr_cluster_publicly_accesible()
result = check.execute()
assert len(result) == 1
assert result[0].region == AWS_REGION
assert result[0].resource_id == cluster_id
assert result[0].resource_arn == cluster_arn
assert result[0].status == "FAIL"
assert (
result[0].status_extended
== f"EMR Cluster {cluster_id} is publicly accessible through the following Security Groups: Slaves Nodes {slave_expected_public_sgs}"
)
@mock_ec2
def test_clusters_master_public_slave_private_two_sg(self):
# EC2 Client
ec2 = resource("ec2", AWS_REGION)
# Create Master Security Group
master_security_group = ec2.create_security_group(
GroupName=str(uuid4()), Description="test-decurity-group"
)
master_security_group.authorize_ingress(
IpProtocol="tcp",
FromPort=0,
ToPort=65535,
CidrIp="0.0.0.0/0",
)
# Create Slave Security Group
slave_security_group = ec2.create_security_group(
GroupName=str(uuid4()), Description="test-decurity-group"
)
slave_security_group.authorize_ingress(
IpProtocol="tcp",
FromPort=0,
ToPort=65535,
CidrIp="10.0.0.0/8",
)
# EMR Client
emr_client = mock.MagicMock
cluster_name = "test-cluster"
cluster_id = "j-XWO1UKVCC6FCV"
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
emr_client.clusters = {
"test-cluster": Cluster(
id=cluster_id,
arn=cluster_arn,
name=cluster_name,
status=ClusterStatus.RUNNING,
region=AWS_REGION,
master_public_dns_name="test.amazonaws.com",
public=True,
master=Node(
security_group_id=master_security_group.id,
additional_security_groups_id=[master_security_group.id],
),
slave=Node(
security_group_id=slave_security_group.id,
additional_security_groups_id=[slave_security_group.id],
),
)
}
master_expected_public_sgs = [
master_security_group.id,
master_security_group.id,
]
from providers.aws.services.ec2.ec2_service import EC2
with mock.patch(
"providers.aws.services.emr.emr_service.EMR",
new=emr_client,
), mock.patch(
"providers.aws.lib.audit_info.audit_info.current_audit_info",
self.set_mocked_audit_info(),
), mock.patch(
"providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client",
new=EC2(self.set_mocked_audit_info()),
):
# Test Check
from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import (
emr_cluster_publicly_accesible,
)
check = emr_cluster_publicly_accesible()
result = check.execute()
assert len(result) == 1
assert result[0].region == AWS_REGION
assert result[0].resource_id == cluster_id
assert result[0].resource_arn == cluster_arn
assert result[0].status == "FAIL"
assert (
result[0].status_extended
== f"EMR Cluster {cluster_id} is publicly accessible through the following Security Groups: Master Node {master_expected_public_sgs}"
)

View File

@@ -0,0 +1,166 @@
import threading
from enum import Enum
from pydantic import BaseModel
from lib.logger import logger
from providers.aws.aws_provider import generate_regional_clients
################## EMR
class EMR:
def __init__(self, audit_info):
self.service = "emr"
self.session = audit_info.audit_session
self.audited_account = audit_info.audited_account
self.regional_clients = generate_regional_clients(self.service, audit_info)
self.clusters = {}
self.block_public_access_configuration = {}
self.__threading_call__(self.__list_clusters__)
self.__threading_call__(self.__describe_cluster__)
self.__threading_call__(self.__get_block_public_access_configuration__)
def __get_session__(self):
return self.session
def __threading_call__(self, call):
threads = []
for regional_client in self.regional_clients.values():
threads.append(threading.Thread(target=call, args=(regional_client,)))
for t in threads:
t.start()
for t in threads:
t.join()
def __list_clusters__(self, regional_client):
logger.info("EMR - Listing Clusters...")
try:
list_clusters_paginator = regional_client.get_paginator("list_clusters")
for page in list_clusters_paginator.paginate():
for cluster in page["Clusters"]:
cluster_name = cluster["Name"]
cluster_id = cluster["Id"]
cluster_arn = cluster["ClusterArn"]
cluster_status = cluster["Status"]["State"]
self.clusters[cluster_id] = Cluster(
id=cluster_id,
name=cluster_name,
arn=cluster_arn,
status=cluster_status,
region=regional_client.region,
)
except Exception as error:
logger.error(
f"{regional_client.region} --"
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
f" {error}"
)
def __describe_cluster__(self, regional_client):
logger.info("EMR - Describing Clusters...")
try:
for cluster in self.clusters.values():
if cluster.region == regional_client.region:
describe_cluster_parameters = {"ClusterId": cluster.id}
cluster_info = regional_client.describe_cluster(
**describe_cluster_parameters
)
# Master Node Security Groups
master_node_security_group = cluster_info["Cluster"][
"Ec2InstanceAttributes"
]["EmrManagedMasterSecurityGroup"]
master_node_additional_security_groups = cluster_info["Cluster"][
"Ec2InstanceAttributes"
]["AdditionalMasterSecurityGroups"]
self.clusters[cluster.id].master = Node(
security_group_id=master_node_security_group,
additional_security_groups_id=master_node_additional_security_groups,
)
# Slave Node Security Groups
slave_node_security_group = cluster_info["Cluster"][
"Ec2InstanceAttributes"
]["EmrManagedSlaveSecurityGroup"]
slave_node_additional_security_groups = cluster_info["Cluster"][
"Ec2InstanceAttributes"
]["AdditionalSlaveSecurityGroups"]
self.clusters[cluster.id].slave = Node(
security_group_id=slave_node_security_group,
additional_security_groups_id=slave_node_additional_security_groups,
)
# Save MasterPublicDnsName
master_public_dns_name = cluster_info["Cluster"][
"MasterPublicDnsName"
]
self.clusters[
cluster.id
].master_public_dns_name = master_public_dns_name
# Set cluster Public/Private
# Public EMR cluster have their DNS ending with .amazonaws.com
# while private ones have format of ip-xxx-xx-xx.us-east-1.compute.internal.
if ".amazonaws.com" in master_public_dns_name:
self.clusters[cluster.id].public = True
except Exception as error:
logger.error(
f"{regional_client.region} --"
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
f" {error}"
)
def __get_block_public_access_configuration__(self, regional_client):
"""Returns the Amazon EMR block public access configuration for your Amazon Web Services account in the current Region."""
logger.info("EMR - Getting Block Public Access Configuration...")
try:
block_public_access_configuration = (
regional_client.get_block_public_access_configuration()
)
self.block_public_access_configuration[
regional_client.region
] = BlockPublicAccessConfiguration(
block_public_security_group_rules=block_public_access_configuration[
"BlockPublicAccessConfiguration"
]["BlockPublicSecurityGroupRules"]
)
except Exception as error:
logger.error(
f"{regional_client.region} --"
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
f" {error}"
)
class BlockPublicAccessConfiguration(BaseModel):
block_public_security_group_rules: bool
class ClusterStatus(Enum):
STARTING = "STARTING"
BOOTSTRAPPING = "BOOTSTRAPPING"
RUNNING = "RUNNING"
WAITING = "WAITING"
TERMINATING = "TERMINATING"
TERMINATED = "TERMINATED"
TERMINATED_WITH_ERRORS = "TERMINATED_WITH_ERRORS"
class Node(BaseModel):
security_group_id: str = ""
additional_security_groups_id: list[str] = []
class Cluster(BaseModel):
id: str
name: str
status: ClusterStatus
arn: str
region: str
master: Node = Node()
slave: Node = Node()
master_public_dns_name: str = ""
public: bool = False

View File

@@ -0,0 +1,136 @@
from datetime import datetime
from unittest.mock import patch
import botocore
from boto3 import client, session
from moto import mock_emr
from moto.core import DEFAULT_ACCOUNT_ID
from providers.aws.lib.audit_info.models import AWS_Audit_Info
from providers.aws.services.emr.emr_service import EMR, ClusterStatus
# Mock Test Region
AWS_REGION = "eu-west-1"
# Mocking Access Analyzer Calls
make_api_call = botocore.client.BaseClient._make_api_call
def mock_make_api_call(self, operation_name, kwarg):
"""We have to mock every AWS API call using Boto3"""
if operation_name == "GetBlockPublicAccessConfiguration":
return {
"BlockPublicAccessConfiguration": {
"BlockPublicSecurityGroupRules": True,
"PermittedPublicSecurityGroupRuleRanges": [
{"MinRange": 0, "MaxRange": 65535},
],
},
"BlockPublicAccessConfigurationMetadata": {
"CreationDateTime": datetime(2015, 1, 1),
"CreatedByArn": "test-arn",
},
}
return make_api_call(self, operation_name, kwarg)
# Mock generate_regional_clients()
def mock_generate_regional_clients(service, audit_info):
regional_client = audit_info.audit_session.client(service, region_name=AWS_REGION)
regional_client.region = AWS_REGION
return {AWS_REGION: regional_client}
@patch(
"providers.aws.services.emr.emr_service.generate_regional_clients",
new=mock_generate_regional_clients,
)
@patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call)
class Test_EMR_Service:
def set_mocked_audit_info(self):
audit_info = AWS_Audit_Info(
original_session=None,
audit_session=session.Session(
profile_name=None,
botocore_session=None,
),
audited_account=DEFAULT_ACCOUNT_ID,
audited_user_id=None,
audited_partition="aws",
audited_identity_arn=None,
profile=None,
profile_region=None,
credentials=None,
assumed_role_info=None,
audited_regions=None,
organizations_metadata=None,
)
return audit_info
# Test EMR Client
@mock_emr
def test__get_client__(self):
emr = EMR(self.set_mocked_audit_info())
assert emr.regional_clients[AWS_REGION].__class__.__name__ == "EMR"
# Test EMR Session
@mock_emr
def test__get_session__(self):
emr = EMR(self.set_mocked_audit_info())
assert emr.session.__class__.__name__ == "Session"
# Test EMR Service
@mock_emr
def test__get_service__(self):
emr = EMR(self.set_mocked_audit_info())
assert emr.service == "emr"
# Test __list_clusters__ and __describe_cluster__
@mock_emr
def test__list_clusters__(self):
# Create EMR Cluster
emr_client = client("emr", region_name=AWS_REGION)
cluster_name = "test-cluster"
run_job_flow_args = dict(
Instances={
"InstanceCount": 3,
"KeepJobFlowAliveWhenNoSteps": True,
"MasterInstanceType": "c3.medium",
"Placement": {"AvailabilityZone": "us-east-1a"},
"SlaveInstanceType": "c3.xlarge",
},
JobFlowRole="EMR_EC2_DefaultRole",
LogUri="s3://mybucket/log",
Name=cluster_name,
ServiceRole="EMR_DefaultRole",
VisibleToAllUsers=True,
)
cluster_id = emr_client.run_job_flow(**run_job_flow_args)["JobFlowId"]
# EMR Class
emr = EMR(self.set_mocked_audit_info())
assert len(emr.clusters) == 1
assert emr.clusters[cluster_id].id == cluster_id
assert emr.clusters[cluster_id].name == cluster_name
assert emr.clusters[cluster_id].status == ClusterStatus.WAITING
assert (
emr.clusters[cluster_id].arn
== f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_id}"
)
assert emr.clusters[cluster_id].region == AWS_REGION
assert (
emr.clusters[cluster_id].master_public_dns_name
== "ec2-184-0-0-1.us-west-1.compute.amazonaws.com"
)
assert emr.clusters[cluster_id].public
@mock_emr
def test__get_block_public_access_configuration__(self):
emr = EMR(self.set_mocked_audit_info())
assert len(emr.block_public_access_configuration) == 1
assert emr.block_public_access_configuration[
AWS_REGION
].block_public_security_group_rules