mirror of
https://github.com/ghndrx/prowler.git
synced 2026-02-11 07:15:15 +00:00
feat(EMR): Service and checks (#1486)
Co-authored-by: Sergio Garcia <38561120+sergargar@users.noreply.github.com>
This commit is contained in:
@@ -1,27 +1,48 @@
|
||||
import ipaddress
|
||||
from typing import Any
|
||||
|
||||
|
||||
################## Security Groups
|
||||
# Check if the security group ingress rule has public access to the check_ports using the protocol
|
||||
def check_security_group(ingress_rule: Any, protocol: str, ports: list = []) -> bool:
|
||||
public_IPv4 = "0.0.0.0/0"
|
||||
public_IPv6 = "::/0"
|
||||
"""
|
||||
Check if the security group ingress rule has public access to the check_ports using the protocol
|
||||
|
||||
@param ingress_rule: AWS Security Group IpPermissions Ingress Rule
|
||||
{
|
||||
'FromPort': 123,
|
||||
'IpProtocol': 'string',
|
||||
'IpRanges': [
|
||||
{
|
||||
'CidrIp': 'string',
|
||||
'Description': 'string'
|
||||
},
|
||||
],
|
||||
'Ipv6Ranges': [
|
||||
{
|
||||
'CidrIpv6': 'string',
|
||||
'Description': 'string'
|
||||
},
|
||||
],
|
||||
'ToPort': 123,
|
||||
}
|
||||
|
||||
@param procotol: Protocol to check.
|
||||
|
||||
|
||||
@param ports: List of ports to check. (Default: [])
|
||||
"""
|
||||
|
||||
# Check for all traffic ingress rules regardless of the protocol
|
||||
if ingress_rule["IpProtocol"] == "-1" and (
|
||||
(
|
||||
"0.0.0.0/0" in str(ingress_rule["IpRanges"])
|
||||
or "::/0" in str(ingress_rule["Ipv6Ranges"])
|
||||
)
|
||||
):
|
||||
return True
|
||||
if ingress_rule["IpProtocol"] == "-1":
|
||||
for ip_ingress_rule in ingress_rule["IpRanges"]:
|
||||
if _is_cidr_public(ip_ingress_rule["CidrIp"]):
|
||||
return True
|
||||
for ip_ingress_rule in ingress_rule["Ipv6Ranges"]:
|
||||
if _is_cidr_public(ip_ingress_rule["CidrIp"]):
|
||||
return True
|
||||
|
||||
# Check for specific ports in ingress rules
|
||||
if "FromPort" in ingress_rule:
|
||||
# All ports
|
||||
if ingress_rule["FromPort"] == 0 and ingress_rule["ToPort"] == 65535:
|
||||
return True
|
||||
|
||||
# If there is a port range
|
||||
if ingress_rule["FromPort"] != ingress_rule["ToPort"]:
|
||||
# Calculate port range, adding 1
|
||||
@@ -35,14 +56,49 @@ def check_security_group(ingress_rule: Any, protocol: str, ports: list = []) ->
|
||||
ingress_port_range.append(int(ingress_rule["FromPort"]))
|
||||
|
||||
# Test Security Group
|
||||
for port in ports:
|
||||
if (
|
||||
(
|
||||
public_IPv4 in str(ingress_rule["IpRanges"])
|
||||
or public_IPv6 in str(ingress_rule["Ipv6Ranges"])
|
||||
)
|
||||
and port in ingress_port_range
|
||||
and ingress_rule["IpProtocol"] == protocol
|
||||
):
|
||||
return True
|
||||
# IPv4
|
||||
for ip_ingress_rule in ingress_rule["IpRanges"]:
|
||||
if _is_cidr_public(ip_ingress_rule["CidrIp"]):
|
||||
# If there are input ports to check
|
||||
if ports:
|
||||
for port in ports:
|
||||
if (
|
||||
port in ingress_port_range
|
||||
and ingress_rule["IpProtocol"] == protocol
|
||||
):
|
||||
return True
|
||||
else:
|
||||
return True
|
||||
|
||||
# IPv6
|
||||
for ip_ingress_rule in ingress_rule["Ipv6Ranges"]:
|
||||
if _is_cidr_public(ip_ingress_rule["CidrIp"]):
|
||||
# If there are input ports to check
|
||||
if ports:
|
||||
for port in ports:
|
||||
if (
|
||||
port in ingress_port_range
|
||||
and ingress_rule["IpProtocol"] == protocol
|
||||
):
|
||||
return True
|
||||
else:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _is_cidr_public(cidr: str) -> bool:
|
||||
"""
|
||||
Check if an input CIDR is public
|
||||
|
||||
@param cidr: CIDR 10.22.33.44/8
|
||||
"""
|
||||
public_IPv4 = "0.0.0.0/0"
|
||||
public_IPv6 = "::/0"
|
||||
# Workaround until this issue is fixed
|
||||
# PR https://github.com/python/cpython/pull/97733
|
||||
# Issue https://github.com/python/cpython/issues/82836
|
||||
if cidr in (public_IPv4, public_IPv6):
|
||||
return True
|
||||
|
||||
return ipaddress.ip_network(cidr).is_global
|
||||
|
||||
21
providers/aws/services/ec2/lib/security_groups_test.py
Normal file
21
providers/aws/services/ec2/lib/security_groups_test.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import pytest
|
||||
|
||||
from providers.aws.services.ec2.lib.security_groups import _is_cidr_public
|
||||
|
||||
|
||||
class Test_security_groups:
|
||||
def test__is_cidr_public_Public_IP(self):
|
||||
cidr = "0.0.0.0/0"
|
||||
assert _is_cidr_public(cidr)
|
||||
|
||||
def test__is_cidr_public_Private_IP(self):
|
||||
cidr = "10.0.0.0/8"
|
||||
assert not _is_cidr_public(cidr)
|
||||
|
||||
def test__is_cidr_public_Bad_Private_IP(self):
|
||||
cidr = "10.0.0.0/0"
|
||||
with pytest.raises(ValueError) as ex:
|
||||
_is_cidr_public(cidr)
|
||||
|
||||
assert ex.type == ValueError
|
||||
assert ex.match(f"{cidr} has host bits set")
|
||||
0
providers/aws/services/emr/__init__.py
Normal file
0
providers/aws/services/emr/__init__.py
Normal file
@@ -1,55 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
# use this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations under the License.
|
||||
|
||||
CHECK_ID_extra7176="7.176"
|
||||
CHECK_TITLE_extra7176="[extra7176] EMR Cluster without Public IP"
|
||||
CHECK_SCORED_extra7176="NOT_SCORED"
|
||||
CHECK_CIS_LEVEL_extra7176="EXTRA"
|
||||
CHECK_SEVERITY_extra7176="Medium"
|
||||
CHECK_ASFF_TYPE_extra7176="AwsEMR"
|
||||
CHECK_ALTERNATE_check7176="extra7176"
|
||||
CHECK_SERVICENAME_extra7176="emr"
|
||||
CHECK_RISK_extra7176='EMR Cluster should not have Public IP'
|
||||
CHECK_REMEDIATION_extra7176='Only make acceptable EMR clusters public'
|
||||
CHECK_DOC_extra7176='https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html'
|
||||
CHECK_CAF_EPIC_extra7176='Infrastructure Security'
|
||||
|
||||
extra7176(){
|
||||
# Public EMR cluster have their DNS ending with .amazonaws.com while private ones have format of ip-xxx-xx-xx.us-east-1.compute.internal.
|
||||
for regx in ${REGIONS}; do
|
||||
# List only EMR clusters with the following states: STARTING, BOOTSTRAPPING, RUNNING, WAITING, TERMINATING
|
||||
# [NOT TERMINATED AND TERMINATED_WITH_ERRORS]
|
||||
LIST_OF_CLUSTERS=$("${AWSCLI}" emr list-clusters ${PROFILE_OPT} --region "${regx}" --query 'Clusters[?(Status.State!=`TERMINATED` && Status.State!=`TERMINATED_WITH_ERRORS`)].Id' --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${LIST_OF_CLUSTERS}"; then
|
||||
textInfo "${regx}: Access Denied trying to list clusters of emr" "${regx}"
|
||||
continue
|
||||
fi
|
||||
if [[ "${LIST_OF_CLUSTERS}" ]]
|
||||
then
|
||||
for cluster_id in ${LIST_OF_CLUSTERS}; do
|
||||
master_public_dns=$("${AWSCLI}" emr describe-cluster ${PROFILE_OPT} --cluster-id "${cluster_id}" --query 'Cluster.MasterPublicDnsName' --region "${regx}" --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_public_dns}"; then
|
||||
textInfo "${regx}: Access Denied trying to describe emr cluster" "${regx}" "${cluster_id}"
|
||||
continue
|
||||
fi
|
||||
if [[ $master_public_dns != None && $master_public_dns != *.internal ]];then
|
||||
textFail "${regx}: EMR Cluster ${cluster_id} has a Public IP" "${regx}" "${cluster_id}"
|
||||
else
|
||||
textPass "${regx}: EMR Cluster ${cluster_id} has not a Public IP" "${regx}" "${cluster_id}"
|
||||
fi
|
||||
done
|
||||
else
|
||||
textInfo "${regx}: No EMR Clusters found" "${regx}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
# use this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations under the License.
|
||||
|
||||
|
||||
CHECK_ID_extra7177="7.177"
|
||||
CHECK_TITLE_extra7177="[extra7177] Publicly accessible EMR Cluster"
|
||||
CHECK_SCORED_extra7177="NOT_SCORED"
|
||||
CHECK_CIS_LEVEL_extra7177="EXTRA"
|
||||
CHECK_SEVERITY_extra7177="High"
|
||||
CHECK_ASFF_TYPE_extra7177="AwsEMR"
|
||||
CHECK_ALTERNATE_check7177="extra7177"
|
||||
CHECK_SERVICENAME_extra7177="emr"
|
||||
CHECK_RISK_extra7177='EMR Clusters should not be publicly accessible'
|
||||
CHECK_REMEDIATION_extra7177='Only make acceptable EMR clusters public'
|
||||
CHECK_DOC_extra7177='https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html'
|
||||
CHECK_CAF_EPIC_extra7177='Infrastructure Security'
|
||||
|
||||
extra7177(){
|
||||
for regx in ${REGIONS}; do
|
||||
# List only EMR clusters with the following states: STARTING, BOOTSTRAPPING, RUNNING, WAITING, TERMINATING
|
||||
# [NOT TERMINATED AND TERMINATED_WITH_ERRORS]
|
||||
LIST_OF_CLUSTERS=$("${AWSCLI}" emr list-clusters ${PROFILE_OPT} --region "${regx}" --query 'Clusters[?(Status.State!=`TERMINATED` && Status.State!=`TERMINATED_WITH_ERRORS`)].Id' --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${LIST_OF_CLUSTERS}"; then
|
||||
textInfo "${regx}: Access Denied trying to list EMR clusters" "${regx}"
|
||||
continue
|
||||
fi
|
||||
if [[ "${LIST_OF_CLUSTERS}" ]]
|
||||
then
|
||||
for cluster_id in ${LIST_OF_CLUSTERS}; do
|
||||
master_public_dns=$("${AWSCLI}" emr describe-cluster ${PROFILE_OPT} --cluster-id "${cluster_id}" --query 'Cluster.MasterPublicDnsName' --region "${regx}" --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_public_dns}"; then
|
||||
textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}"
|
||||
continue
|
||||
fi
|
||||
if [[ $master_public_dns != None && $master_public_dns != *.internal ]];then
|
||||
# If EMR cluster is Public, it is required to check their Security Groups for the Master, the Slaves and the additional ones
|
||||
|
||||
# Retrive EMR Master Node Security Groups rules
|
||||
master_node_sg=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.EmrManagedMasterSecurityGroup' --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_node_sg}"; then
|
||||
textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}"
|
||||
continue
|
||||
fi
|
||||
master_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${master_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${master_node_sg_internet_open}"; then
|
||||
textInfo "$regx: Access Denied trying to describe security groups" "$regx"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Retrive EMR Slave Node Security Groups rules
|
||||
slave_node_sg=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.EmrManagedSlaveSecurityGroup' --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg}"; then
|
||||
textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}"
|
||||
continue
|
||||
fi
|
||||
slave_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${slave_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg_internet_open}"; then
|
||||
textInfo "$regx: Access Denied trying to describe security groups" "$regx"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Retrive EMR Additional Master node Security Groups rules
|
||||
additional_master_node_sg_list=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.AdditionalMasterSecurityGroups' --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg}"; then
|
||||
textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}"
|
||||
continue
|
||||
fi
|
||||
local additional_master_node_sg_internet_open_list
|
||||
if [[ "${additional_master_node_sg_list}" != "None" ]]; then
|
||||
for additional_master_node_sg in ${additional_master_node_sg_list}; do
|
||||
additional_master_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${additional_master_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg_internet_open}"; then
|
||||
textInfo "$regx: Access Denied trying to describe security groups" "$regx"
|
||||
continue
|
||||
fi
|
||||
# Store additional master node security groups that allows access from the internet
|
||||
additional_master_node_sg_internet_open_list+=( "${additional_master_node_sg_internet_open}" )
|
||||
done
|
||||
fi
|
||||
|
||||
# Retrive EMR Additional Slave node Security Groups rules
|
||||
additional_slave_node_sg_list=$("${AWSCLI}" emr describe-cluster --cluster-id "${cluster_id}" ${PROFILE_OPT} --region "${regx}" --query 'Cluster.Ec2InstanceAttributes.AdditionalSlaveSecurityGroups' --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${slave_node_sg}"; then
|
||||
textInfo "${regx}: Access Denied trying to describe EMR cluster" "${regx}" "${cluster_id}"
|
||||
continue
|
||||
fi
|
||||
local additional_slave_node_sg_internet_open_list
|
||||
if [[ "${additional_slave_node_sg_list}" != "None" ]]; then
|
||||
for additional_slave_node_sg in ${additional_master_node_sg_list}; do
|
||||
additional_slave_node_sg_internet_open=$("${AWSCLI}" ec2 describe-security-groups --group-ids "${additional_slave_node_sg}" --query 'SecurityGroups[?length(IpPermissions[?(contains(IpRanges[].CidrIp, `0.0.0.0/0`) || contains(Ipv6Ranges[].CidrIpv6, `::/0`))]) > `0`].{GroupId:GroupId}' ${PROFILE_OPT} --region "${regx}" --output text 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${additional_slave_node_sg_internet_open}"; then
|
||||
textInfo "$regx: Access Denied trying to describe security groups" "$regx"
|
||||
continue
|
||||
fi
|
||||
# Store additional slave node security groups that allows access from the internet
|
||||
additional_slave_node_sg_internet_open_list+=( "${additional_slave_node_sg_internet_open}" )
|
||||
done
|
||||
fi
|
||||
|
||||
# Check if EMR Cluster is publicly accessible through a Security Group
|
||||
if [[ -n "${master_node_sg_internet_open}" || -n "${slave_node_sg_internet_open}" || "${#additional_master_node_sg_internet_open_list[@]}" -ne 0 || "${#additional_slave_node_sg_internet_open_list[@]}" -ne 0 ]]; then
|
||||
textFail "${regx}: EMR Cluster ${cluster_id} is publicly accessible through the following Security Groups: Master Node ${master_node_sg_internet_open} ${additional_master_node_sg_internet_open_list[*]} -- Slaves Nodes ${slave_node_sg_internet_open} ${additional_slave_node_sg_internet_open_list[*]}" "${regx}" "${cluster_id}"
|
||||
else
|
||||
textPass "${regx}: EMR Cluster ${cluster_id} is not publicly accessible" "${regx}" "${cluster_id}"
|
||||
fi
|
||||
else
|
||||
textPass "${regx}: EMR Cluster ${cluster_id} is not publicly accessible" "${regx}" "${cluster_id}"
|
||||
fi
|
||||
done
|
||||
else
|
||||
textInfo "${regx}: No EMR Clusters found" "${regx}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
# use this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations under the License.
|
||||
|
||||
|
||||
CHECK_ID_extra7178="7.178"
|
||||
CHECK_TITLE_extra7178="[extra7178] EMR Account Public Access Block enabled"
|
||||
CHECK_SCORED_extra7178="NOT_SCORED"
|
||||
CHECK_CIS_LEVEL_extra7178="EXTRA"
|
||||
CHECK_SEVERITY_extra7178="High"
|
||||
CHECK_ASFF_TYPE_extra7178="AwsEMR"
|
||||
CHECK_ALTERNATE_check7178="extra7178"
|
||||
CHECK_SERVICENAME_extra7178="emr"
|
||||
CHECK_RISK_extra7178='EMR Clusters must have Account Public Access Block enabled'
|
||||
CHECK_REMEDIATION_extra7178='Enable EMR Account Public Access Block'
|
||||
CHECK_DOC_extra7178='https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html'
|
||||
CHECK_CAF_EPIC_extra7178='Infrastructure Security'
|
||||
|
||||
extra7178(){
|
||||
for regx in ${REGIONS}; do
|
||||
block_public_access=$("${AWSCLI}" emr get-block-public-access-configuration ${PROFILE_OPT} --region "${regx}" --query 'BlockPublicAccessConfiguration.BlockPublicSecurityGroupRules' --output json 2>&1)
|
||||
if grep -q -E 'AccessDenied|UnauthorizedOperation|AuthorizationError' <<< "${block_public_access}"; then
|
||||
textInfo "${regx}: Access Denied trying to get block public access configuration for EMR clusters" "${regx}"
|
||||
continue
|
||||
fi
|
||||
if [[ "${block_public_access}" == "true" ]]; then
|
||||
textPass "${regx}: EMR Account has Block Public Access enabled" "${regx}"
|
||||
else
|
||||
textFail "${regx}: EMR Account has Block Public Access disabled" "${regx}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
4
providers/aws/services/emr/emr_client.py
Normal file
4
providers/aws/services/emr/emr_client.py
Normal file
@@ -0,0 +1,4 @@
|
||||
from providers.aws.lib.audit_info.audit_info import current_audit_info
|
||||
from providers.aws.services.emr.emr_service import EMR
|
||||
|
||||
emr_client = EMR(current_audit_info)
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"Provider": "aws",
|
||||
"CheckID": "emr_cluster_account_public_block_enabled",
|
||||
"CheckTitle": "EMR Account Public Access Block enabled.",
|
||||
"CheckType": [],
|
||||
"ServiceName": "emr",
|
||||
"SubServiceName": "",
|
||||
"ResourceIdTemplate": "arn:aws:emr:region:account-id",
|
||||
"Severity": "high",
|
||||
"ResourceType": "AwsEMR",
|
||||
"Description": "EMR Account Public Access Block enabled.",
|
||||
"Risk": "EMR Clusters must have Account Public Access Block enabled.",
|
||||
"RelatedUrl": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html",
|
||||
"Remediation": {
|
||||
"Code": {
|
||||
"CLI": "",
|
||||
"NativeIaC": "",
|
||||
"Other": "https://github.com/cloudmatos/matos/tree/master/remediations/aws/emr/block-emr-public-access",
|
||||
"Terraform": ""
|
||||
},
|
||||
"Recommendation": {
|
||||
"Text": "Enable EMR Account Public Access Block.",
|
||||
"Url": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html"
|
||||
}
|
||||
},
|
||||
"Categories": [],
|
||||
"Tags": {
|
||||
"Tag1Key": "value",
|
||||
"Tag2Key": "value"
|
||||
},
|
||||
"DependsOn": [],
|
||||
"RelatedTo": [],
|
||||
"Notes": "",
|
||||
"Compliance": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
from lib.check.models import Check, Check_Report
|
||||
from providers.aws.services.emr.emr_client import emr_client
|
||||
|
||||
|
||||
class emr_cluster_account_public_block_enabled(Check):
|
||||
def execute(self):
|
||||
findings = []
|
||||
for region in emr_client.block_public_access_configuration:
|
||||
report = Check_Report(self.metadata)
|
||||
report.region = region
|
||||
report.resource_id = emr_client.audited_account
|
||||
|
||||
if emr_client.block_public_access_configuration[
|
||||
region
|
||||
].block_public_security_group_rules:
|
||||
report.status = "PASS"
|
||||
report.status_extended = "EMR Account has Block Public Access enabled"
|
||||
else:
|
||||
report.status = "FAIL"
|
||||
report.status_extended = "EMR Account has Block Public Access disabled"
|
||||
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
@@ -0,0 +1,67 @@
|
||||
from unittest import mock
|
||||
|
||||
from moto.core import DEFAULT_ACCOUNT_ID
|
||||
|
||||
from providers.aws.services.emr.emr_service import BlockPublicAccessConfiguration
|
||||
|
||||
AWS_REGION = "eu-west-1"
|
||||
|
||||
|
||||
class Test_emr_cluster_account_public_block_enabled:
|
||||
def test_account_public_block_enabled(self):
|
||||
emr_client = mock.MagicMock
|
||||
emr_client.audited_account = DEFAULT_ACCOUNT_ID
|
||||
emr_client.block_public_access_configuration = {
|
||||
AWS_REGION: BlockPublicAccessConfiguration(
|
||||
block_public_security_group_rules=True
|
||||
)
|
||||
}
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_account_public_block_enabled.emr_cluster_account_public_block_enabled import (
|
||||
emr_cluster_account_public_block_enabled,
|
||||
)
|
||||
|
||||
check = emr_cluster_account_public_block_enabled()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].region == AWS_REGION
|
||||
assert result[0].resource_id == DEFAULT_ACCOUNT_ID
|
||||
assert result[0].status == "PASS"
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== "EMR Account has Block Public Access enabled"
|
||||
)
|
||||
|
||||
def test_account_public_block_disabled(self):
|
||||
emr_client = mock.MagicMock
|
||||
emr_client.audited_account = DEFAULT_ACCOUNT_ID
|
||||
emr_client.block_public_access_configuration = {
|
||||
AWS_REGION: BlockPublicAccessConfiguration(
|
||||
block_public_security_group_rules=False
|
||||
)
|
||||
}
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_account_public_block_enabled.emr_cluster_account_public_block_enabled import (
|
||||
emr_cluster_account_public_block_enabled,
|
||||
)
|
||||
|
||||
check = emr_cluster_account_public_block_enabled()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].region == AWS_REGION
|
||||
assert result[0].resource_id == DEFAULT_ACCOUNT_ID
|
||||
assert result[0].status == "FAIL"
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== "EMR Account has Block Public Access disabled"
|
||||
)
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"Provider": "aws",
|
||||
"CheckID": "emr_cluster_master_nodes_no_public_ip",
|
||||
"CheckTitle": "EMR Cluster without Public IP.",
|
||||
"CheckType": [],
|
||||
"ServiceName": "emr",
|
||||
"SubServiceName": "",
|
||||
"ResourceIdTemplate": "arn:aws:emr:region:account-id",
|
||||
"Severity": "medium",
|
||||
"ResourceType": "AwsEMR",
|
||||
"Description": "EMR Cluster without Public IP.",
|
||||
"Risk": "EMR Cluster should not have Public IP.",
|
||||
"RelatedUrl": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html",
|
||||
"Remediation": {
|
||||
"Code": {
|
||||
"CLI": "",
|
||||
"NativeIaC": "",
|
||||
"Other": "",
|
||||
"Terraform": ""
|
||||
},
|
||||
"Recommendation": {
|
||||
"Text": "Only make acceptable EMR clusters public.",
|
||||
"Url": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html"
|
||||
}
|
||||
},
|
||||
"Categories": [],
|
||||
"Tags": {
|
||||
"Tag1Key": "value",
|
||||
"Tag2Key": "value"
|
||||
},
|
||||
"DependsOn": [],
|
||||
"RelatedTo": [],
|
||||
"Notes": "",
|
||||
"Compliance": []
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
from lib.check.models import Check, Check_Report
|
||||
from providers.aws.services.emr.emr_client import emr_client
|
||||
from providers.aws.services.emr.emr_service import ClusterStatus
|
||||
|
||||
|
||||
class emr_cluster_master_nodes_no_public_ip(Check):
|
||||
def execute(self):
|
||||
findings = []
|
||||
for cluster in emr_client.clusters.values():
|
||||
if cluster.status not in (
|
||||
ClusterStatus.TERMINATED,
|
||||
ClusterStatus.TERMINATED_WITH_ERRORS,
|
||||
):
|
||||
report = Check_Report(self.metadata)
|
||||
report.region = cluster.region
|
||||
report.resource_id = cluster.id
|
||||
report.resource_arn = cluster.arn
|
||||
|
||||
if cluster.public:
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"EMR Cluster {cluster.id} has a Public IP"
|
||||
else:
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"EMR Cluster {cluster.id} has not a Public IP"
|
||||
)
|
||||
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
@@ -0,0 +1,169 @@
|
||||
from unittest import mock
|
||||
|
||||
from moto.core import DEFAULT_ACCOUNT_ID
|
||||
|
||||
from providers.aws.services.emr.emr_service import Cluster, ClusterStatus
|
||||
|
||||
AWS_REGION = "eu-west-1"
|
||||
|
||||
|
||||
class Test_emr_cluster_master_nodes_no_public_ip:
|
||||
def test_no_clusters(self):
|
||||
emr_client = mock.MagicMock
|
||||
emr_client.clusters = {}
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import (
|
||||
emr_cluster_master_nodes_no_public_ip,
|
||||
)
|
||||
|
||||
check = emr_cluster_master_nodes_no_public_ip()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
def test_cluster_public_running(self):
|
||||
emr_client = mock.MagicMock
|
||||
cluster_name = "test-cluster"
|
||||
cluster_id = "j-XWO1UKVCC6FCV"
|
||||
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
|
||||
emr_client.clusters = {
|
||||
"test-cluster": Cluster(
|
||||
id=cluster_id,
|
||||
arn=cluster_arn,
|
||||
name=cluster_name,
|
||||
status=ClusterStatus.RUNNING,
|
||||
region=AWS_REGION,
|
||||
master_public_dns_name="test.amazonaws.com",
|
||||
public=True,
|
||||
)
|
||||
}
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import (
|
||||
emr_cluster_master_nodes_no_public_ip,
|
||||
)
|
||||
|
||||
check = emr_cluster_master_nodes_no_public_ip()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].region == AWS_REGION
|
||||
assert result[0].resource_id == cluster_id
|
||||
assert result[0].resource_arn == cluster_arn
|
||||
assert result[0].status == "FAIL"
|
||||
assert (
|
||||
result[0].status_extended == f"EMR Cluster {cluster_id} has a Public IP"
|
||||
)
|
||||
|
||||
def test_cluster_private_running(self):
|
||||
emr_client = mock.MagicMock
|
||||
cluster_name = "test-cluster"
|
||||
cluster_id = "j-XWO1UKVCC6FCV"
|
||||
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
|
||||
emr_client.clusters = {
|
||||
"test-cluster": Cluster(
|
||||
id=cluster_id,
|
||||
arn=cluster_arn,
|
||||
name=cluster_name,
|
||||
status=ClusterStatus.RUNNING,
|
||||
region=AWS_REGION,
|
||||
master_public_dns_name="compute.internal",
|
||||
public=False,
|
||||
)
|
||||
}
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import (
|
||||
emr_cluster_master_nodes_no_public_ip,
|
||||
)
|
||||
|
||||
check = emr_cluster_master_nodes_no_public_ip()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].region == AWS_REGION
|
||||
assert result[0].resource_id == cluster_id
|
||||
assert result[0].resource_arn == cluster_arn
|
||||
assert result[0].status == "PASS"
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== f"EMR Cluster {cluster_id} has not a Public IP"
|
||||
)
|
||||
|
||||
def test_cluster_public_terminated(self):
|
||||
emr_client = mock.MagicMock
|
||||
cluster_name = "test-cluster"
|
||||
cluster_id = "j-XWO1UKVCC6FCV"
|
||||
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
|
||||
emr_client.clusters = {
|
||||
"test-cluster": Cluster(
|
||||
id=cluster_id,
|
||||
arn=cluster_arn,
|
||||
name=cluster_name,
|
||||
status=ClusterStatus.TERMINATED,
|
||||
region=AWS_REGION,
|
||||
master_public_dns_name="test.amazonaws.com",
|
||||
public=True,
|
||||
)
|
||||
}
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import (
|
||||
emr_cluster_master_nodes_no_public_ip,
|
||||
)
|
||||
|
||||
check = emr_cluster_master_nodes_no_public_ip()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
def test_cluster_private_bootstrapping(self):
|
||||
emr_client = mock.MagicMock
|
||||
cluster_name = "test-cluster"
|
||||
cluster_id = "j-XWO1UKVCC6FCV"
|
||||
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
|
||||
emr_client.clusters = {
|
||||
"test-cluster": Cluster(
|
||||
id=cluster_id,
|
||||
arn=cluster_arn,
|
||||
name=cluster_name,
|
||||
status=ClusterStatus.BOOTSTRAPPING,
|
||||
region=AWS_REGION,
|
||||
master_public_dns_name="compute.internal",
|
||||
public=False,
|
||||
)
|
||||
}
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_master_nodes_no_public_ip.emr_cluster_master_nodes_no_public_ip import (
|
||||
emr_cluster_master_nodes_no_public_ip,
|
||||
)
|
||||
|
||||
check = emr_cluster_master_nodes_no_public_ip()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].region == AWS_REGION
|
||||
assert result[0].resource_id == cluster_id
|
||||
assert result[0].resource_arn == cluster_arn
|
||||
assert result[0].status == "PASS"
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== f"EMR Cluster {cluster_id} has not a Public IP"
|
||||
)
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"Provider": "aws",
|
||||
"CheckID": "emr_cluster_publicly_accesible",
|
||||
"CheckTitle": "Publicly accessible EMR Cluster.",
|
||||
"CheckType": [],
|
||||
"ServiceName": "emr",
|
||||
"SubServiceName": "",
|
||||
"ResourceIdTemplate": "arn:aws:emr:region:account-id",
|
||||
"Severity": "medium",
|
||||
"ResourceType": "AwsEMR",
|
||||
"Description": "Publicly accessible EMR Cluster.",
|
||||
"Risk": "EMR Clusters should not be publicly accessible.",
|
||||
"RelatedUrl": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html",
|
||||
"Remediation": {
|
||||
"Code": {
|
||||
"CLI": "",
|
||||
"NativeIaC": "",
|
||||
"Other": "",
|
||||
"Terraform": "https://docs.bridgecrew.io/docs/ensure-that-amazon-emr-clusters-security-groups-are-not-open-to-the-world#terraform"
|
||||
},
|
||||
"Recommendation": {
|
||||
"Text": "Only make acceptable EMR clusters public.",
|
||||
"Url": "https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-block-public-access.html"
|
||||
}
|
||||
},
|
||||
"Categories": [],
|
||||
"Tags": {
|
||||
"Tag1Key": "value",
|
||||
"Tag2Key": "value"
|
||||
},
|
||||
"DependsOn": [],
|
||||
"RelatedTo": [],
|
||||
"Notes": "",
|
||||
"Compliance": []
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lib.check.models import Check, Check_Report
|
||||
from providers.aws.services.ec2.ec2_client import ec2_client
|
||||
from providers.aws.services.ec2.lib.security_groups import check_security_group
|
||||
from providers.aws.services.emr.emr_client import emr_client
|
||||
from providers.aws.services.emr.emr_service import ClusterStatus
|
||||
|
||||
|
||||
class emr_cluster_publicly_accesible(Check):
|
||||
def execute(self):
|
||||
findings = []
|
||||
for cluster in emr_client.clusters.values():
|
||||
if cluster.status not in (
|
||||
ClusterStatus.TERMINATED,
|
||||
ClusterStatus.TERMINATED_WITH_ERRORS,
|
||||
):
|
||||
report = Check_Report(self.metadata)
|
||||
report.region = cluster.region
|
||||
report.resource_id = cluster.id
|
||||
report.resource_arn = cluster.arn
|
||||
|
||||
report.status = "PASS"
|
||||
report.status_extended = (
|
||||
f"EMR Cluster {cluster.id} is not publicly accessible"
|
||||
)
|
||||
# If EMR cluster is Public, it is required to check
|
||||
# their Security Groups for the Master,
|
||||
# the Slaves and the additional ones
|
||||
if cluster.public:
|
||||
|
||||
# Check Public Master Security Groups
|
||||
master_node_sg_groups = deepcopy(
|
||||
cluster.master.additional_security_groups_id
|
||||
)
|
||||
master_node_sg_groups.append(cluster.master.security_group_id)
|
||||
|
||||
master_public_security_groups = []
|
||||
for master_sg in master_node_sg_groups:
|
||||
master_sg_public = False
|
||||
for sg in ec2_client.security_groups:
|
||||
if sg.id == master_sg:
|
||||
for ingress_rule in sg.ingress_rules:
|
||||
if check_security_group(ingress_rule, -1):
|
||||
master_sg_public = True
|
||||
break
|
||||
if master_sg_public:
|
||||
master_public_security_groups.append(sg.id)
|
||||
break
|
||||
|
||||
# Check Public Slave Security Groups
|
||||
slave_node_sg_groups = deepcopy(
|
||||
cluster.slave.additional_security_groups_id
|
||||
)
|
||||
slave_node_sg_groups.append(cluster.slave.security_group_id)
|
||||
|
||||
slave_public_security_groups = []
|
||||
for slave_sg in slave_node_sg_groups:
|
||||
slave_sg_public = False
|
||||
for sg in ec2_client.security_groups:
|
||||
if sg.id == slave_sg:
|
||||
for ingress_rule in sg.ingress_rules:
|
||||
if check_security_group(ingress_rule, -1):
|
||||
slave_sg_public = True
|
||||
break
|
||||
if slave_sg_public:
|
||||
slave_public_security_groups.append(sg.id)
|
||||
break
|
||||
|
||||
if master_public_security_groups or slave_public_security_groups:
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"EMR Cluster {cluster.id} is publicly accessible through the following Security Groups:"
|
||||
report.status_extended += (
|
||||
f" Master Node {master_public_security_groups}"
|
||||
if master_public_security_groups
|
||||
else ""
|
||||
)
|
||||
report.status_extended += (
|
||||
f" Slaves Nodes {slave_public_security_groups}"
|
||||
if slave_public_security_groups
|
||||
else ""
|
||||
)
|
||||
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
@@ -0,0 +1,361 @@
|
||||
from unittest import mock
|
||||
from uuid import uuid4
|
||||
|
||||
from boto3 import resource, session
|
||||
from moto import mock_ec2
|
||||
from moto.core import DEFAULT_ACCOUNT_ID
|
||||
|
||||
from providers.aws.lib.audit_info.audit_info import AWS_Audit_Info
|
||||
from providers.aws.services.emr.emr_service import Cluster, ClusterStatus, Node
|
||||
|
||||
AWS_REGION = "eu-west-1"
|
||||
|
||||
|
||||
class Test_emr_cluster_publicly_accesible:
|
||||
# Mocked Audit Info
|
||||
def set_mocked_audit_info(self):
|
||||
audit_info = AWS_Audit_Info(
|
||||
original_session=None,
|
||||
audit_session=session.Session(
|
||||
profile_name=None,
|
||||
botocore_session=None,
|
||||
),
|
||||
audited_account=None,
|
||||
audited_user_id=None,
|
||||
audited_partition="aws",
|
||||
audited_identity_arn=None,
|
||||
profile=None,
|
||||
profile_region=None,
|
||||
credentials=None,
|
||||
assumed_role_info=None,
|
||||
audited_regions=None,
|
||||
organizations_metadata=None,
|
||||
)
|
||||
return audit_info
|
||||
|
||||
def test_no_clusters(self):
|
||||
# EMR Client
|
||||
emr_client = mock.MagicMock
|
||||
emr_client.clusters = {}
|
||||
# EC2 Client
|
||||
ec2_client = mock.MagicMock
|
||||
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
), mock.patch(
|
||||
"providers.aws.services.ec2.ec2_service.EC2",
|
||||
new=ec2_client,
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import (
|
||||
emr_cluster_publicly_accesible,
|
||||
)
|
||||
|
||||
check = emr_cluster_publicly_accesible()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
@mock_ec2
|
||||
def test_clusters_master_public_sg(self):
|
||||
# EC2 Client
|
||||
ec2 = resource("ec2", AWS_REGION)
|
||||
# Create Security Group
|
||||
master_security_group = ec2.create_security_group(
|
||||
GroupName=str(uuid4()), Description="test-decurity-group"
|
||||
)
|
||||
master_security_group.authorize_ingress(
|
||||
IpProtocol="tcp",
|
||||
FromPort=0,
|
||||
ToPort=65535,
|
||||
CidrIp="0.0.0.0/0",
|
||||
)
|
||||
|
||||
# EMR Client
|
||||
emr_client = mock.MagicMock
|
||||
cluster_name = "test-cluster"
|
||||
cluster_id = "j-XWO1UKVCC6FCV"
|
||||
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
|
||||
emr_client.clusters = {
|
||||
"test-cluster": Cluster(
|
||||
id=cluster_id,
|
||||
arn=cluster_arn,
|
||||
name=cluster_name,
|
||||
status=ClusterStatus.RUNNING,
|
||||
region=AWS_REGION,
|
||||
master_public_dns_name="test.amazonaws.com",
|
||||
public=True,
|
||||
master=Node(
|
||||
security_group_id=master_security_group.id,
|
||||
additional_security_groups_id=[],
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
master_expected_public_sgs = [master_security_group.id]
|
||||
|
||||
from providers.aws.services.ec2.ec2_service import EC2
|
||||
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
), mock.patch(
|
||||
"providers.aws.lib.audit_info.audit_info.current_audit_info",
|
||||
self.set_mocked_audit_info(),
|
||||
), mock.patch(
|
||||
"providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client",
|
||||
new=EC2(self.set_mocked_audit_info()),
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import (
|
||||
emr_cluster_publicly_accesible,
|
||||
)
|
||||
|
||||
check = emr_cluster_publicly_accesible()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].region == AWS_REGION
|
||||
assert result[0].resource_id == cluster_id
|
||||
assert result[0].resource_arn == cluster_arn
|
||||
assert result[0].status == "FAIL"
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== f"EMR Cluster {cluster_id} is publicly accessible through the following Security Groups: Master Node {master_expected_public_sgs}"
|
||||
)
|
||||
|
||||
@mock_ec2
|
||||
def test_clusters_master_private_sg(self):
|
||||
# EC2 Client
|
||||
ec2 = resource("ec2", AWS_REGION)
|
||||
# Create Security Group
|
||||
master_security_group = ec2.create_security_group(
|
||||
GroupName=str(uuid4()), Description="test-decurity-group"
|
||||
)
|
||||
master_security_group.authorize_ingress(
|
||||
IpProtocol="tcp",
|
||||
FromPort=0,
|
||||
ToPort=65535,
|
||||
CidrIp="10.0.0.0/8",
|
||||
)
|
||||
|
||||
# EMR Client
|
||||
emr_client = mock.MagicMock
|
||||
cluster_name = "test-cluster"
|
||||
cluster_id = "j-XWO1UKVCC6FCV"
|
||||
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
|
||||
emr_client.clusters = {
|
||||
"test-cluster": Cluster(
|
||||
id=cluster_id,
|
||||
arn=cluster_arn,
|
||||
name=cluster_name,
|
||||
status=ClusterStatus.RUNNING,
|
||||
region=AWS_REGION,
|
||||
master_public_dns_name="test.amazonaws.com",
|
||||
public=True,
|
||||
master=Node(
|
||||
security_group_id=master_security_group.id,
|
||||
additional_security_groups_id=[],
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
from providers.aws.services.ec2.ec2_service import EC2
|
||||
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
), mock.patch(
|
||||
"providers.aws.lib.audit_info.audit_info.current_audit_info",
|
||||
self.set_mocked_audit_info(),
|
||||
), mock.patch(
|
||||
"providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client",
|
||||
new=EC2(self.set_mocked_audit_info()),
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import (
|
||||
emr_cluster_publicly_accesible,
|
||||
)
|
||||
|
||||
check = emr_cluster_publicly_accesible()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].region == AWS_REGION
|
||||
assert result[0].resource_id == cluster_id
|
||||
assert result[0].resource_arn == cluster_arn
|
||||
assert result[0].status == "PASS"
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== f"EMR Cluster {cluster_id} is not publicly accessible"
|
||||
)
|
||||
|
||||
@mock_ec2
|
||||
def test_clusters_master_private_slave_public_sg(self):
|
||||
# EC2 Client
|
||||
ec2 = resource("ec2", AWS_REGION)
|
||||
# Create Master Security Group
|
||||
master_security_group = ec2.create_security_group(
|
||||
GroupName=str(uuid4()), Description="test-decurity-group"
|
||||
)
|
||||
master_security_group.authorize_ingress(
|
||||
IpProtocol="tcp",
|
||||
FromPort=0,
|
||||
ToPort=65535,
|
||||
CidrIp="10.0.0.0/8",
|
||||
)
|
||||
|
||||
# Create Slave Security Group
|
||||
slave_security_group = ec2.create_security_group(
|
||||
GroupName=str(uuid4()), Description="test-decurity-group"
|
||||
)
|
||||
slave_security_group.authorize_ingress(
|
||||
IpProtocol="tcp",
|
||||
FromPort=0,
|
||||
ToPort=65535,
|
||||
CidrIp="0.0.0.0/0",
|
||||
)
|
||||
|
||||
# EMR Client
|
||||
emr_client = mock.MagicMock
|
||||
cluster_name = "test-cluster"
|
||||
cluster_id = "j-XWO1UKVCC6FCV"
|
||||
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
|
||||
emr_client.clusters = {
|
||||
"test-cluster": Cluster(
|
||||
id=cluster_id,
|
||||
arn=cluster_arn,
|
||||
name=cluster_name,
|
||||
status=ClusterStatus.RUNNING,
|
||||
region=AWS_REGION,
|
||||
master_public_dns_name="test.amazonaws.com",
|
||||
public=True,
|
||||
master=Node(
|
||||
security_group_id=master_security_group.id,
|
||||
additional_security_groups_id=[],
|
||||
),
|
||||
slave=Node(
|
||||
security_group_id=slave_security_group.id,
|
||||
additional_security_groups_id=[],
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
slave_expected_public_sgs = [slave_security_group.id]
|
||||
|
||||
from providers.aws.services.ec2.ec2_service import EC2
|
||||
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
), mock.patch(
|
||||
"providers.aws.lib.audit_info.audit_info.current_audit_info",
|
||||
self.set_mocked_audit_info(),
|
||||
), mock.patch(
|
||||
"providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client",
|
||||
new=EC2(self.set_mocked_audit_info()),
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import (
|
||||
emr_cluster_publicly_accesible,
|
||||
)
|
||||
|
||||
check = emr_cluster_publicly_accesible()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].region == AWS_REGION
|
||||
assert result[0].resource_id == cluster_id
|
||||
assert result[0].resource_arn == cluster_arn
|
||||
assert result[0].status == "FAIL"
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== f"EMR Cluster {cluster_id} is publicly accessible through the following Security Groups: Slaves Nodes {slave_expected_public_sgs}"
|
||||
)
|
||||
|
||||
@mock_ec2
|
||||
def test_clusters_master_public_slave_private_two_sg(self):
|
||||
# EC2 Client
|
||||
ec2 = resource("ec2", AWS_REGION)
|
||||
# Create Master Security Group
|
||||
master_security_group = ec2.create_security_group(
|
||||
GroupName=str(uuid4()), Description="test-decurity-group"
|
||||
)
|
||||
master_security_group.authorize_ingress(
|
||||
IpProtocol="tcp",
|
||||
FromPort=0,
|
||||
ToPort=65535,
|
||||
CidrIp="0.0.0.0/0",
|
||||
)
|
||||
|
||||
# Create Slave Security Group
|
||||
slave_security_group = ec2.create_security_group(
|
||||
GroupName=str(uuid4()), Description="test-decurity-group"
|
||||
)
|
||||
slave_security_group.authorize_ingress(
|
||||
IpProtocol="tcp",
|
||||
FromPort=0,
|
||||
ToPort=65535,
|
||||
CidrIp="10.0.0.0/8",
|
||||
)
|
||||
|
||||
# EMR Client
|
||||
emr_client = mock.MagicMock
|
||||
cluster_name = "test-cluster"
|
||||
cluster_id = "j-XWO1UKVCC6FCV"
|
||||
cluster_arn = f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_name}"
|
||||
emr_client.clusters = {
|
||||
"test-cluster": Cluster(
|
||||
id=cluster_id,
|
||||
arn=cluster_arn,
|
||||
name=cluster_name,
|
||||
status=ClusterStatus.RUNNING,
|
||||
region=AWS_REGION,
|
||||
master_public_dns_name="test.amazonaws.com",
|
||||
public=True,
|
||||
master=Node(
|
||||
security_group_id=master_security_group.id,
|
||||
additional_security_groups_id=[master_security_group.id],
|
||||
),
|
||||
slave=Node(
|
||||
security_group_id=slave_security_group.id,
|
||||
additional_security_groups_id=[slave_security_group.id],
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
master_expected_public_sgs = [
|
||||
master_security_group.id,
|
||||
master_security_group.id,
|
||||
]
|
||||
|
||||
from providers.aws.services.ec2.ec2_service import EC2
|
||||
|
||||
with mock.patch(
|
||||
"providers.aws.services.emr.emr_service.EMR",
|
||||
new=emr_client,
|
||||
), mock.patch(
|
||||
"providers.aws.lib.audit_info.audit_info.current_audit_info",
|
||||
self.set_mocked_audit_info(),
|
||||
), mock.patch(
|
||||
"providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible.ec2_client",
|
||||
new=EC2(self.set_mocked_audit_info()),
|
||||
):
|
||||
# Test Check
|
||||
from providers.aws.services.emr.emr_cluster_publicly_accesible.emr_cluster_publicly_accesible import (
|
||||
emr_cluster_publicly_accesible,
|
||||
)
|
||||
|
||||
check = emr_cluster_publicly_accesible()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].region == AWS_REGION
|
||||
assert result[0].resource_id == cluster_id
|
||||
assert result[0].resource_arn == cluster_arn
|
||||
assert result[0].status == "FAIL"
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== f"EMR Cluster {cluster_id} is publicly accessible through the following Security Groups: Master Node {master_expected_public_sgs}"
|
||||
)
|
||||
166
providers/aws/services/emr/emr_service.py
Normal file
166
providers/aws/services/emr/emr_service.py
Normal file
@@ -0,0 +1,166 @@
|
||||
import threading
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from lib.logger import logger
|
||||
from providers.aws.aws_provider import generate_regional_clients
|
||||
|
||||
|
||||
################## EMR
|
||||
class EMR:
|
||||
def __init__(self, audit_info):
|
||||
self.service = "emr"
|
||||
self.session = audit_info.audit_session
|
||||
self.audited_account = audit_info.audited_account
|
||||
self.regional_clients = generate_regional_clients(self.service, audit_info)
|
||||
self.clusters = {}
|
||||
self.block_public_access_configuration = {}
|
||||
self.__threading_call__(self.__list_clusters__)
|
||||
self.__threading_call__(self.__describe_cluster__)
|
||||
self.__threading_call__(self.__get_block_public_access_configuration__)
|
||||
|
||||
def __get_session__(self):
|
||||
return self.session
|
||||
|
||||
def __threading_call__(self, call):
|
||||
threads = []
|
||||
for regional_client in self.regional_clients.values():
|
||||
threads.append(threading.Thread(target=call, args=(regional_client,)))
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
def __list_clusters__(self, regional_client):
|
||||
logger.info("EMR - Listing Clusters...")
|
||||
try:
|
||||
list_clusters_paginator = regional_client.get_paginator("list_clusters")
|
||||
for page in list_clusters_paginator.paginate():
|
||||
for cluster in page["Clusters"]:
|
||||
cluster_name = cluster["Name"]
|
||||
cluster_id = cluster["Id"]
|
||||
cluster_arn = cluster["ClusterArn"]
|
||||
cluster_status = cluster["Status"]["State"]
|
||||
|
||||
self.clusters[cluster_id] = Cluster(
|
||||
id=cluster_id,
|
||||
name=cluster_name,
|
||||
arn=cluster_arn,
|
||||
status=cluster_status,
|
||||
region=regional_client.region,
|
||||
)
|
||||
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} --"
|
||||
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
|
||||
f" {error}"
|
||||
)
|
||||
|
||||
def __describe_cluster__(self, regional_client):
|
||||
logger.info("EMR - Describing Clusters...")
|
||||
try:
|
||||
for cluster in self.clusters.values():
|
||||
if cluster.region == regional_client.region:
|
||||
describe_cluster_parameters = {"ClusterId": cluster.id}
|
||||
cluster_info = regional_client.describe_cluster(
|
||||
**describe_cluster_parameters
|
||||
)
|
||||
|
||||
# Master Node Security Groups
|
||||
master_node_security_group = cluster_info["Cluster"][
|
||||
"Ec2InstanceAttributes"
|
||||
]["EmrManagedMasterSecurityGroup"]
|
||||
master_node_additional_security_groups = cluster_info["Cluster"][
|
||||
"Ec2InstanceAttributes"
|
||||
]["AdditionalMasterSecurityGroups"]
|
||||
self.clusters[cluster.id].master = Node(
|
||||
security_group_id=master_node_security_group,
|
||||
additional_security_groups_id=master_node_additional_security_groups,
|
||||
)
|
||||
|
||||
# Slave Node Security Groups
|
||||
slave_node_security_group = cluster_info["Cluster"][
|
||||
"Ec2InstanceAttributes"
|
||||
]["EmrManagedSlaveSecurityGroup"]
|
||||
slave_node_additional_security_groups = cluster_info["Cluster"][
|
||||
"Ec2InstanceAttributes"
|
||||
]["AdditionalSlaveSecurityGroups"]
|
||||
self.clusters[cluster.id].slave = Node(
|
||||
security_group_id=slave_node_security_group,
|
||||
additional_security_groups_id=slave_node_additional_security_groups,
|
||||
)
|
||||
|
||||
# Save MasterPublicDnsName
|
||||
master_public_dns_name = cluster_info["Cluster"][
|
||||
"MasterPublicDnsName"
|
||||
]
|
||||
self.clusters[
|
||||
cluster.id
|
||||
].master_public_dns_name = master_public_dns_name
|
||||
# Set cluster Public/Private
|
||||
# Public EMR cluster have their DNS ending with .amazonaws.com
|
||||
# while private ones have format of ip-xxx-xx-xx.us-east-1.compute.internal.
|
||||
if ".amazonaws.com" in master_public_dns_name:
|
||||
self.clusters[cluster.id].public = True
|
||||
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} --"
|
||||
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
|
||||
f" {error}"
|
||||
)
|
||||
|
||||
def __get_block_public_access_configuration__(self, regional_client):
|
||||
"""Returns the Amazon EMR block public access configuration for your Amazon Web Services account in the current Region."""
|
||||
logger.info("EMR - Getting Block Public Access Configuration...")
|
||||
try:
|
||||
block_public_access_configuration = (
|
||||
regional_client.get_block_public_access_configuration()
|
||||
)
|
||||
|
||||
self.block_public_access_configuration[
|
||||
regional_client.region
|
||||
] = BlockPublicAccessConfiguration(
|
||||
block_public_security_group_rules=block_public_access_configuration[
|
||||
"BlockPublicAccessConfiguration"
|
||||
]["BlockPublicSecurityGroupRules"]
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} --"
|
||||
f" {error.__class__.__name__}[{error.__traceback__.tb_lineno}]:"
|
||||
f" {error}"
|
||||
)
|
||||
|
||||
|
||||
class BlockPublicAccessConfiguration(BaseModel):
|
||||
block_public_security_group_rules: bool
|
||||
|
||||
|
||||
class ClusterStatus(Enum):
|
||||
STARTING = "STARTING"
|
||||
BOOTSTRAPPING = "BOOTSTRAPPING"
|
||||
RUNNING = "RUNNING"
|
||||
WAITING = "WAITING"
|
||||
TERMINATING = "TERMINATING"
|
||||
TERMINATED = "TERMINATED"
|
||||
TERMINATED_WITH_ERRORS = "TERMINATED_WITH_ERRORS"
|
||||
|
||||
|
||||
class Node(BaseModel):
|
||||
security_group_id: str = ""
|
||||
additional_security_groups_id: list[str] = []
|
||||
|
||||
|
||||
class Cluster(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
status: ClusterStatus
|
||||
arn: str
|
||||
region: str
|
||||
master: Node = Node()
|
||||
slave: Node = Node()
|
||||
master_public_dns_name: str = ""
|
||||
public: bool = False
|
||||
136
providers/aws/services/emr/emr_service_test.py
Normal file
136
providers/aws/services/emr/emr_service_test.py
Normal file
@@ -0,0 +1,136 @@
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
import botocore
|
||||
from boto3 import client, session
|
||||
from moto import mock_emr
|
||||
from moto.core import DEFAULT_ACCOUNT_ID
|
||||
|
||||
from providers.aws.lib.audit_info.models import AWS_Audit_Info
|
||||
from providers.aws.services.emr.emr_service import EMR, ClusterStatus
|
||||
|
||||
# Mock Test Region
|
||||
AWS_REGION = "eu-west-1"
|
||||
|
||||
|
||||
# Mocking Access Analyzer Calls
|
||||
make_api_call = botocore.client.BaseClient._make_api_call
|
||||
|
||||
|
||||
def mock_make_api_call(self, operation_name, kwarg):
|
||||
"""We have to mock every AWS API call using Boto3"""
|
||||
if operation_name == "GetBlockPublicAccessConfiguration":
|
||||
return {
|
||||
"BlockPublicAccessConfiguration": {
|
||||
"BlockPublicSecurityGroupRules": True,
|
||||
"PermittedPublicSecurityGroupRuleRanges": [
|
||||
{"MinRange": 0, "MaxRange": 65535},
|
||||
],
|
||||
},
|
||||
"BlockPublicAccessConfigurationMetadata": {
|
||||
"CreationDateTime": datetime(2015, 1, 1),
|
||||
"CreatedByArn": "test-arn",
|
||||
},
|
||||
}
|
||||
|
||||
return make_api_call(self, operation_name, kwarg)
|
||||
|
||||
|
||||
# Mock generate_regional_clients()
|
||||
def mock_generate_regional_clients(service, audit_info):
|
||||
regional_client = audit_info.audit_session.client(service, region_name=AWS_REGION)
|
||||
regional_client.region = AWS_REGION
|
||||
return {AWS_REGION: regional_client}
|
||||
|
||||
|
||||
@patch(
|
||||
"providers.aws.services.emr.emr_service.generate_regional_clients",
|
||||
new=mock_generate_regional_clients,
|
||||
)
|
||||
@patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call)
|
||||
class Test_EMR_Service:
|
||||
def set_mocked_audit_info(self):
|
||||
audit_info = AWS_Audit_Info(
|
||||
original_session=None,
|
||||
audit_session=session.Session(
|
||||
profile_name=None,
|
||||
botocore_session=None,
|
||||
),
|
||||
audited_account=DEFAULT_ACCOUNT_ID,
|
||||
audited_user_id=None,
|
||||
audited_partition="aws",
|
||||
audited_identity_arn=None,
|
||||
profile=None,
|
||||
profile_region=None,
|
||||
credentials=None,
|
||||
assumed_role_info=None,
|
||||
audited_regions=None,
|
||||
organizations_metadata=None,
|
||||
)
|
||||
return audit_info
|
||||
|
||||
# Test EMR Client
|
||||
@mock_emr
|
||||
def test__get_client__(self):
|
||||
emr = EMR(self.set_mocked_audit_info())
|
||||
assert emr.regional_clients[AWS_REGION].__class__.__name__ == "EMR"
|
||||
|
||||
# Test EMR Session
|
||||
@mock_emr
|
||||
def test__get_session__(self):
|
||||
emr = EMR(self.set_mocked_audit_info())
|
||||
assert emr.session.__class__.__name__ == "Session"
|
||||
|
||||
# Test EMR Service
|
||||
@mock_emr
|
||||
def test__get_service__(self):
|
||||
emr = EMR(self.set_mocked_audit_info())
|
||||
assert emr.service == "emr"
|
||||
|
||||
# Test __list_clusters__ and __describe_cluster__
|
||||
@mock_emr
|
||||
def test__list_clusters__(self):
|
||||
# Create EMR Cluster
|
||||
emr_client = client("emr", region_name=AWS_REGION)
|
||||
cluster_name = "test-cluster"
|
||||
run_job_flow_args = dict(
|
||||
Instances={
|
||||
"InstanceCount": 3,
|
||||
"KeepJobFlowAliveWhenNoSteps": True,
|
||||
"MasterInstanceType": "c3.medium",
|
||||
"Placement": {"AvailabilityZone": "us-east-1a"},
|
||||
"SlaveInstanceType": "c3.xlarge",
|
||||
},
|
||||
JobFlowRole="EMR_EC2_DefaultRole",
|
||||
LogUri="s3://mybucket/log",
|
||||
Name=cluster_name,
|
||||
ServiceRole="EMR_DefaultRole",
|
||||
VisibleToAllUsers=True,
|
||||
)
|
||||
cluster_id = emr_client.run_job_flow(**run_job_flow_args)["JobFlowId"]
|
||||
# EMR Class
|
||||
emr = EMR(self.set_mocked_audit_info())
|
||||
|
||||
assert len(emr.clusters) == 1
|
||||
assert emr.clusters[cluster_id].id == cluster_id
|
||||
assert emr.clusters[cluster_id].name == cluster_name
|
||||
assert emr.clusters[cluster_id].status == ClusterStatus.WAITING
|
||||
assert (
|
||||
emr.clusters[cluster_id].arn
|
||||
== f"arn:aws:elasticmapreduce:{AWS_REGION}:{DEFAULT_ACCOUNT_ID}:cluster/{cluster_id}"
|
||||
)
|
||||
assert emr.clusters[cluster_id].region == AWS_REGION
|
||||
assert (
|
||||
emr.clusters[cluster_id].master_public_dns_name
|
||||
== "ec2-184-0-0-1.us-west-1.compute.amazonaws.com"
|
||||
)
|
||||
assert emr.clusters[cluster_id].public
|
||||
|
||||
@mock_emr
|
||||
def test__get_block_public_access_configuration__(self):
|
||||
emr = EMR(self.set_mocked_audit_info())
|
||||
|
||||
assert len(emr.block_public_access_configuration) == 1
|
||||
assert emr.block_public_access_configuration[
|
||||
AWS_REGION
|
||||
].block_public_security_group_rules
|
||||
Reference in New Issue
Block a user