Integration with Yelp detect-secrets

This commit is contained in:
Toni de la Fuente
2019-06-25 08:28:50 -04:00
parent f54bc4238e
commit ea6d9c93fc
9 changed files with 267 additions and 43 deletions

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2018) by Toni de la Fuente
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
@@ -11,48 +11,58 @@
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
CHECK_ID_extra741="7.41"
CHECK_TITLE_extra741="[extra741] Find keys in EC2 UserData (Not Scored) (Not part of CIS benchmark)"
CHECK_TITLE_extra741="[extra741] Find secrets in EC2 User Data (Not Scored) (Not part of CIS benchmark)"
CHECK_SCORED_extra741="NOT_SCORED"
CHECK_TYPE_extra741="EXTRA"
CHECK_ALTERNATE_check741="extra741"
extra741(){
textInfo "Looking for keys in EC2 User Data in instances across all regions... (max 100 instances per region use -m to increase it) "
SECRETS_TEMP_FOLDER="$PROWLER_DIR/secrets-$ACCOUNT_NUM"
if [[ ! -d $SECRETS_TEMP_FOLDER ]]; then
# this folder is deleted once this check is finished
mkdir $SECRETS_TEMP_FOLDER
fi
textInfo "Looking for secrets in EC2 User Data in instances across all regions... (max 100 instances per region use -m to increase it) "
for regx in $REGIONS; do
LIST_OF_EC2_INSTANCES=$($AWSCLI ec2 describe-instances $PROFILE_OPT --region $regx --query Reservations[*].Instances[*].InstanceId --output text --max-items $MAXITEMS | grep -v None)
if [[ $LIST_OF_EC2_INSTANCES ]];then
for instance in $LIST_OF_EC2_INSTANCES; do
USERDATA_FILE=$instance-userdata.decoded
USERDATA=$($AWSCLI ec2 describe-instance-attribute --attribute userData --query UserData.Value $PROFILE_OPT --region $regx --instance-id $instance --output text | decode_report > $USERDATA_FILE)
if [ -s $USERDATA_FILE ];then
FILE_FORMAT_ASCII=$(file -b $USERDATA_FILE|grep ASCII)
#FINDINGS=$(grep '[A-Za-z0-9]\{20,40\}' $USERDATA_FILE | grep -i -e key -e secret -e token -e pass - |wc -l|tr -d '\ ')
#FINDINGS=$(grep -i -e key -e secret -e token -e pass $USERDATA_FILE |wc -l|tr -d '\ ')
EC2_USERDATA_FILE="$SECRETS_TEMP_FOLDER/extra741-$instance-userData.decoded"
EC2_USERDATA=$($AWSCLI ec2 describe-instance-attribute --attribute userData $PROFILE_OPT --region $regx --instance-id $instance --query UserData.Value --output text| grep -v ^None | decode_report > $EC2_USERDATA_FILE)
if [ -s $EC2_USERDATA_FILE ];then
FILE_FORMAT_ASCII=$(file -b $EC2_USERDATA_FILE|grep ASCII)
# This finds ftp or http URLs with credentials and common keywords
FINDINGS=$(egrep -i '[[:alpha:]]*://[[:alnum:]]*:[[:alnum:]]*@.*/|key|secret|token|pass' $USERDATA_FILE |wc -l|tr -d '\ ')
# FINDINGS=$(egrep -i '[[:alpha:]]*://[[:alnum:]]*:[[:alnum:]]*@.*/|key|secret|token|pass' $EC2_USERDATA_FILE |wc -l|tr -d '\ ')
# New implementation using https://github.com/Yelp/detect-secrets
if [[ $FILE_FORMAT_ASCII ]]; then
if [[ $FINDINGS -eq "0" ]]; then
textPass "$regx: No keys found in $instance" "$regx"
FINDINGS=$(secretsDetector file $EC2_USERDATA_FILE)
if [[ $FINDINGS -eq 0 ]]; then
textPass "$regx: No secrets found in $instance" "$regx"
# delete file if nothing interesting is there
rm -f $USERDATA_FILE
rm -f $EC2_USERDATA_FILE
else
textFail "$regx: Found $FINDINGS keys in $instance! Check file $USERDATA_FILE" "$regx"
textFail "$regx: Potential secret found in $instance" "$regx"
# delete file to not leave trace, user must look at the instance User Data
rm -f $EC2_USERDATA_FILE
fi
else
mv $USERDATA_FILE $USERDATA_FILE.gz ; gunzip $USERDATA_FILE.gz
mv $EC2_USERDATA_FILE $EC2_USERDATA_FILE.gz ; gunzip $EC2_USERDATA_FILE.gz
FINDINGS=$(secretsDetector file $EC2_USERDATA_FILE)
if [[ $FINDINGS -eq 0 ]]; then
textPass "$regx: No keys found in $instance" "$regx"
rm -f $USERDATA_FILE.gz
textPass "$regx: No secrets found in $instance User Data" "$regx"
rm -f $EC2_USERDATA_FILE
else
textFail "$regx: Found $FINDINGS keys in $instance! Check file $USERDATA_FILE" "$regx"
textFail "$regx: Potential secret found in $instance" "$regx"
fi
fi
else
textPass "$regx: $instance nothing found" "$regx"
else
textPass "$regx: No secrets found in $instance User Data or it is empty" "$regx"
fi
done
else
textInfo "$regx: No EC2 instances found" "$regx"
fi
done
rm -rf $SECRETS_TEMP_FOLDER
}

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2018) by Toni de la Fuente
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
@@ -11,37 +11,46 @@
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
CHECK_ID_extra742="7.42"
CHECK_TITLE_extra742="[extra742] Find keys in CloudFormation outputs (Not Scored) (Not part of CIS benchmark)"
CHECK_TITLE_extra742="[extra742] Find secrets in CloudFormation outputs (Not Scored) (Not part of CIS benchmark)"
CHECK_SCORED_extra742="NOT_SCORED"
CHECK_TYPE_extra742="EXTRA"
CHECK_ALTERNATE_check742="extra742"
extra742(){
textInfo "Looking for keys in CloudFormation output across all regions... "
SECRETS_TEMP_FOLDER="$PROWLER_DIR/secrets-$ACCOUNT_NUM"
if [[ ! -d $SECRETS_TEMP_FOLDER ]]; then
# this folder is deleted once this check is finished
mkdir $SECRETS_TEMP_FOLDER
fi
textInfo "Looking for secrets in CloudFormation output across all regions... "
for regx in $REGIONS; do
LIST_OF_CFN_STACKS=$($AWSCLI cloudformation describe-stacks $PROFILE_OPT --region $regx --query Stacks[*].[StackName] --output text)
if [[ $LIST_OF_CFN_STACKS ]];then
for stack in $LIST_OF_CFN_STACKS; do
OUTPUTS_FILE=$stack-$regx-output.txt
OUTPUTS=$($AWSCLI $PROFILE_OPT --region $regx cloudformation describe-stacks --query "Stacks[?StackName==\`$stack\`].Outputs[*].[OutputKey,OutputValue]" --output text > $OUTPUTS_FILE)
if [ -s $OUTPUTS ];then
#FINDINGS=$(grep '[A-Za-z0-9]\{20,40\}' $USERDATA_FILE | grep -i -e key -e secret -e token -e pass - |wc -l|tr -d '\ ')
#FINDINGS=$(grep -i -e key -e secret -e token -e pass $USERDATA_FILE |wc -l|tr -d '\ ')
CFN_OUTPUTS_FILE="$SECRETS_TEMP_FOLDER/extra742-$stack-$regx-outputs.txt"
CFN_OUTPUTS=$($AWSCLI $PROFILE_OPT --region $regx cloudformation describe-stacks --query "Stacks[?StackName==\`$stack\`].Outputs[*].[OutputKey,OutputValue]" --output text > $CFN_OUTPUTS_FILE)
if [ -s $CFN_OUTPUTS_FILE ];then
# This finds ftp or http URLs with credentials and common keywords
FINDINGS=$(egrep -i '[[:alpha:]]*://[[:alnum:]]*:[[:alnum:]]*@.*/|key|secret|token|pass' $OUTPUTS_FILE |wc -l|tr -d '\ ')
if [[ $FINDINGS -eq "0" ]]; then
textPass "$regx: No keys found in Stack $stack" "$regx"
# delete file if nothing interesting is there
rm -f $OUTPUTS_FILE
else
textFail "$regx: Found $FINDINGS keys in $stack! Check file $OUTPUTS_FILE" "$regx"
fi
# FINDINGS=$(egrep -i '[[:alpha:]]*://[[:alnum:]]*:[[:alnum:]]*@.*/|key|secret|token|pass' $CFN_OUTPUTS_FILE |wc -l|tr -d '\ ')
# New implementation using https://github.com/Yelp/detect-secrets
FINDINGS=$(secretsDetector file $CFN_OUTPUTS_FILE)
if [[ $FINDINGS -eq 0 ]]; then
textPass "$regx: No secrets found in stack $stack Outputs" "$regx"
# delete file if nothing interesting is there
rm -f $CFN_OUTPUTS_FILE
else
textFail "$regx: Potential secret found in stack $stack Outputs" "$regx"
# delete file to not leave trace, user must look at the CFN Stack
rm -f $CFN_OUTPUTS_FILE
fi
else
textPass "$regx: Stack $stack has not Outputs" "$regx"
textInfo "$regx: CloudFormation stack $stack has not Outputs" "$regx"
fi
done
else
textInfo "$regx: No CloudFormation stacks found" "$regx"
fi
done
rm -rf $SECRETS_TEMP_FOLDER
}

54
checks/check_extra759 Normal file
View File

@@ -0,0 +1,54 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
CHECK_ID_extra759="7.59"
CHECK_TITLE_extra759="[extra759] Find secrets in Lambda functions variables (Not Scored) (Not part of CIS benchmark)"
CHECK_SCORED_extra759="NOT_SCORED"
CHECK_TYPE_extra759="EXTRA"
CHECK_ALTERNATE_check759="extra759"
extra759(){
SECRETS_TEMP_FOLDER="$PROWLER_DIR/secrets-$ACCOUNT_NUM"
if [[ ! -d $SECRETS_TEMP_FOLDER ]]; then
# this folder is deleted once this check is finished
mkdir $SECRETS_TEMP_FOLDER
fi
textInfo "Looking for secrets in Lambda variables across all regions... "
for regx in $REGIONS; do
LIST_OF_FUNCTIONS=$($AWSCLI lambda list-functions $PROFILE_OPT --region $regx --query Functions[*].FunctionName --output text)
if [[ $LIST_OF_FUNCTIONS ]]; then
for lambdafunction in $LIST_OF_FUNCTIONS;do
LAMBDA_FUNCTION_VARIABLES_FILE="$SECRETS_TEMP_FOLDER/extra759-$lambdafunction-$regx-variables.txt"
LAMBDA_FUNCTION_VARIABLES=$($AWSCLI lambda $PROFILE_OPT --region $regx get-function-configuration --function-name $lambdafunction --query 'Environment.Variables' --output text > $LAMBDA_FUNCTION_VARIABLES_FILE)
if [ -s $LAMBDA_FUNCTION_VARIABLES_FILE ];then
# Implementation using https://github.com/Yelp/detect-secrets
FINDINGS=$(secretsDetector file $LAMBDA_FUNCTION_VARIABLES_FILE)
if [[ $FINDINGS -eq 0 ]]; then
textPass "$regx: No secrets found in Lambda function $lambdafunction variables" "$regx"
# delete file if nothing interesting is there
rm -f $LAMBDA_FUNCTION_VARIABLES_FILE
else
textFail "$regx: Potential secret found in Lambda function $lambdafunction variables" "$regx"
# delete file to not leave trace, user must look at the function
rm -f $LAMBDA_FUNCTION_VARIABLES_FILE
fi
else
textInfo "$regx: Lambda function $stalambdafunction has not variables" "$regx"
fi
done
else
textInfo "$regx: No Lambda functions found" "$regx"
fi
done
rm -rf $SECRETS_TEMP_FOLDER
}

55
checks/check_extra760 Normal file
View File

@@ -0,0 +1,55 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
CHECK_ID_extra760="7.60"
CHECK_TITLE_extra760="[extra760] Find secrets in Lambda functions code (Not Scored) (Not part of CIS benchmark)"
CHECK_SCORED_extra760="NOT_SCORED"
CHECK_TYPE_extra760="EXTRA"
CHECK_ALTERNATE_check760="extra760"
extra760(){
SECRETS_TEMP_FOLDER="$PROWLER_DIR/secrets-$ACCOUNT_NUM"
if [[ ! -d $SECRETS_TEMP_FOLDER ]]; then
# this folder is deleted once this check is finished
mkdir $SECRETS_TEMP_FOLDER
fi
textInfo "Looking for secrets in Lambda functions code across all regions... "
textInfo "This check may take a while depending on your functions size! "
for regx in $REGIONS; do
LIST_OF_FUNCTIONS=$($AWSCLI lambda list-functions $PROFILE_OPT --region $regx --query Functions[*].FunctionName --output text)
if [[ $LIST_OF_FUNCTIONS ]]; then
for lambdafunction in $LIST_OF_FUNCTIONS;do
LAMBDA_FUNCTION_FOLDER="$SECRETS_TEMP_FOLDER/extra760-$lambdafunction-$regx"
LAMBDA_FUNCTION_FILE="$lambdafunction-code.zip"
LAMBDA_CODE_LOCATION=$($AWSCLI lambda get-function $PROFILE_OPT --region $regx --function-name $lambdafunction --query Code.Location --output text)
mkdir $LAMBDA_FUNCTION_FOLDER
# DOWNLOAD the code in a zip file
curl -s $LAMBDA_CODE_LOCATION -o $LAMBDA_FUNCTION_FOLDER/$LAMBDA_FUNCTION_FILE
unzip -qq $LAMBDA_FUNCTION_FOLDER/$LAMBDA_FUNCTION_FILE -d $LAMBDA_FUNCTION_FOLDER
FINDINGS=$(secretsDetector folder $LAMBDA_FUNCTION_FOLDER)
if [[ $FINDINGS -eq 0 ]]; then
textPass "$regx: No secrets found in Lambda function $lambdafunction code" "$regx"
# delete files if nothing interesting is there
rm -fr $LAMBDA_FUNCTION_FOLDER
else
textFail "$regx: Potential secret found in Lambda function $lambdafunction code" "$regx"
# delete files to not leave trace, user must look at the function
rm -fr $LAMBDA_FUNCTION_FOLDER
fi
done
else
textInfo "$regx: No Lambda functions found" "$regx"
fi
done
rm -fr $SECRETS_TEMP_FOLDER
}

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2018) by Toni de la Fuente
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
@@ -11,12 +11,17 @@
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
GROUP_ID[11]='keys'
GROUP_ID[11]='secrets'
GROUP_NUMBER[11]='11.0'
GROUP_TITLE[11]='Look for keys secrets or passwords around resources - [keys] **'
GROUP_RUN_BY_DEFAULT[11]='N' # run it when execute_all is called
GROUP_CHECKS[11]='extra741,extra742'
GROUP_TITLE[11]='Look for keys secrets or passwords around resources - [secrets] **'
GROUP_RUN_BY_DEFAULT[11]='N' # but it runs when execute_all is called (default)
GROUP_CHECKS[11]='extra741,extra742,extra759,extra760'
# requires https://github.com/Yelp/detect-secrets
# `pip install detect-secrets`
# Initially:
# - EC2 UserData
# - CloudFormation Outputs
# - Lambda variables
# - Lambda code

View File

@@ -16,3 +16,6 @@ GROUP_NUMBER[7]='7.0'
GROUP_TITLE[7]='Extras - [extras] **********************************************'
GROUP_RUN_BY_DEFAULT[7]='Y' # run it when execute_all is called
GROUP_CHECKS[7]='extra71,extra72,extra73,extra74,extra75,extra76,extra77,extra78,extra79,extra710,extra711,extra712,extra713,extra714,extra715,extra716,extra717,extra718,extra719,extra720,extra721,extra722,extra723,extra724,extra725,extra726,extra727,extra728,extra729,extra730,extra731,extra732,extra733,extra734,extra735,extra736,extra737,extra738,extra739,extra740,extra741,extra742,extra743,extra744,extra745,extra746,extra747,extra748,extra749,extra750,extra751,extra752,extra753,extra754,extra755,extra756,extra757,extra758'
# Extras 759 and 760 (lambda variables and code secrets finder are not included)
# to run detect-secrets use `./prowler -g secrets`

32
include/python_detector Normal file
View File

@@ -0,0 +1,32 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
# detector of python and boto3
pythonDetector(){
PYTHON_BIN=$(which python)
PYTHON_PIP_BOTO3=$(pip list|grep boto3)
if [ -z "${PYTHON_BIN}" ]; then
echo -e "\n$RED ERROR!$NORMAL python not found. Make sure it is installed correctly and in your \$PATH\n"
EXITCODE=1
exit $EXITCODE
else
PYTHON_INSTALLED=1
if [ -z "${PYTHON_PIP_BOTO3}" ]; then
echo -e "\n$RED ERROR!$NORMAL python library boto3 not found. Make sure it is installed correctly\n"
EXITCODE=1
exit $EXITCODE
else
PYTHON_PIP_BOTO3_INSTALLED=1
fi
fi
}

54
include/secrets_detector Normal file
View File

@@ -0,0 +1,54 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2019) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
# detector and configuration of detect-secrets
secretsDetector(){
PYTHON_PIP_DETECTSECRETS=$(which detect-secrets)
if [ -z "${PYTHON_PIP_DETECTSECRETS}" ]; then
echo -e "\n$RED ERROR!$NORMAL python library detect-secrets not found. Make sure it is installed correctly and in your \$PATH\n"
EXITCODE=1
exit $EXITCODE
else
SECRETS_TEMP_FOLDER="$PROWLER_DIR/secrets-$ACCOUNT_NUM"
if [[ ! -d $SECRETS_TEMP_FOLDER ]]; then
mkdir $SECRETS_TEMP_FOLDER
fi
PYTHON_PIP_DETECTSECRETS_INSTALLED=1
# Sets the entropy limit for high entropy base64 strings. Value
# must be between 0.0 and 8.0, defaults is 4.5.
BASE64_LIMIT=3.0
# Sets the entropy limit for high entropy hex strings. Value
# must be between 0.0 and 8.0, defaults is 3.0.
HEX_LIMIT=3.0
case $1 in
file )
# this is to scan a file
detect-secrets scan --hex-limit $HEX_LIMIT --base64-limit $BASE64_LIMIT $2 | \
jq -r '.results[]|.[] | [.line_number, .type]|@csv' | wc -l
#jq -r '.results[] | .[] | "\(.line_number)\t\(.type)"'
# this command must return values in two colums:
# line in file and type of secrets found
;;
string )
# this is to scan a given string
detect-secrets scan --hex-limit $HEX_LIMIT --base64-limit $BASE64_LIMIT --string $2 | \
grep True| wc -l
;;
folder )
# this is to scan a given folder with all lambda files
detect-secrets scan --hex-limit $HEX_LIMIT --base64-limit $BASE64_LIMIT --all-files $2 | \
jq -r '.results[]|.[] | [.line_number, .type]|@csv' | wc -l
;;
esac
fi
}

View File

@@ -32,7 +32,7 @@ OPTRED=""
OPTNORMAL=""
# Set the defaults variables
PROWLER_VERSION=2.0.2
PROWLER_VERSION=2.1.0
PROWLER_DIR=$(dirname "$0")
REGION=""
@@ -163,6 +163,8 @@ done
. $PROWLER_DIR/include/whoami
. $PROWLER_DIR/include/credentials_report
. $PROWLER_DIR/include/scoring
. $PROWLER_DIR/include/python_detector
. $PROWLER_DIR/include/secrets_detector
# Get a list of all available AWS Regions
REGIONS=$($AWSCLI ec2 describe-regions --query 'Regions[].RegionName' \