From 56a4fd813c38db6e382134464ce1421f7efca275 Mon Sep 17 00:00:00 2001 From: Urjit Singh Bhatia Date: Tue, 10 Mar 2020 18:54:32 -0700 Subject: [PATCH 1/5] Support whitelists per check --- include/outputs | 28 +++++++++++++++++++++++----- prowler | 27 ++++++++++++++++++++++++--- whitelist_sample.txt | 4 ++++ 3 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 whitelist_sample.txt diff --git a/include/outputs b/include/outputs index b18ac683..c89350b5 100644 --- a/include/outputs +++ b/include/outputs @@ -104,15 +104,29 @@ textInfo(){ } textFail(){ - FAIL_COUNTER=$((FAIL_COUNTER+1)) - EXITCODE=3 + ## ignore whitelists for current check + level="FAIL" + for i in $IGNORES; do + ignore_value="${i#*${CHECK_NAME}:}" + if [[ $1 =~ ${ignore_value} ]]; then + level="WARNING" + break + fi + done + + # only set non-0 exit code on FAIL mode, WARN is ok + if [[ "$level" == "FAIL" ]]; then + FAIL_COUNTER=$((FAIL_COUNTER+1)) + EXITCODE=3 + fi + if [[ "$MODE" == "csv" ]]; then if [[ $2 ]]; then REPREGION=$2 else REPREGION=$REGION fi - echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}FAIL${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" + echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}${level}${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" elif [[ "$MODE" == "json" ]]; then if [[ $2 ]]; then REPREGION=$2 @@ -128,7 +142,7 @@ textFail(){ --arg ITEM_LEVEL "$ITEM_LEVEL" \ --arg TITLE_ID "$TITLE_ID" \ --arg REPREGION "$REPREGION" \ - --arg TIMESTAMP $(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + --arg TIMESTAMP "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ -n '{ "Profile": $PROFILE, "Account Number": $ACCOUNT_NUM, @@ -142,7 +156,11 @@ textFail(){ "Timestamp": $TIMESTAMP, }' else - echo " $BAD FAIL! $1 $NORMAL" + if [[ "${level}" == "FAIL" ]]; then + echo " $BAD ${level}! $1 $NORMAL" + else + echo " $WARNING ${level}! $1 $NORMAL" + fi fi } diff --git a/prowler b/prowler index a3a8db27..05ee1e00 100755 --- a/prowler +++ b/prowler @@ -47,6 +47,7 @@ EXITCODE=0 SCRIPT_START_TIME=$( date -u +"%Y-%m-%dT%H:%M:%S%z" ) TITLE_ID="" TITLE_TEXT="CALLER ERROR - UNSET TITLE" +WHITELIST_FILE="" # Command usage menu usage(){ @@ -83,12 +84,19 @@ USAGE: (i.e.: ProwlerRole) -T session durantion given to that role credentials in seconds, default 1h (3600) recommended 12h, requires -R and -T (i.e.: 43200) + -w whitelist file. (Lines starting with # are ignored as comments) Format: + # ignore these due to some reason + # check1 checks s3 buckets + : + : + # checkid2 + : -h this help " exit } -while getopts ":hlLkqp:r:c:g:f:m:M:E:enbVsx:A:R:T:" OPTION; do +while getopts ":hlLkqp:r:c:g:f:m:M:E:enbVsx:A:R:T:w:" OPTION; do case $OPTION in h ) usage @@ -160,6 +168,11 @@ while getopts ":hlLkqp:r:c:g:f:m:M:E:enbVsx:A:R:T:" OPTION; do T ) SESSION_DURATION_TO_ASSUME=$OPTARG ;; + w ) + WHITELIST_FILE=$OPTARG + echo "" + echo "$OPTNORMAL Using Whitelist file: $OPTARG" + ;; : ) echo "" echo "$OPTRED ERROR!$OPTNORMAL -$OPTARG requires an argument" @@ -202,6 +215,12 @@ REGIONS=$($AWSCLI ec2 describe-regions --query 'Regions[].RegionName' \ --region $REGION \ --region-names $FILTERREGION) +# Pre-process whitelist file if supplied +if [[ -n "$WHITELIST_FILE" ]]; then + # ignore lines starting with # (comments) + WHITELIST=$(awk '!/^[[:space:]]*#/{print }' <(cat "$WHITELIST_FILE")) +fi + # Load all of the groups of checks inside groups folder named as "groupNumber*" for group in $(ls $PROWLER_DIR/groups/group[0-9]*|grep -v groupN_sample); do . "$group" @@ -266,8 +285,10 @@ execute_check() { saveReport fi fi - show_check_title $1 - $1 + show_check_title "$1" + ignores=$(awk '/${1}/{print}' <(echo "${WHITELIST}")) + # set the custom ignores list for this check + IGNORES="${ignores}" $1 else textFail "ERROR! Use a valid check name (i.e. check41 or extra71)"; exit $EXITCODE diff --git a/whitelist_sample.txt b/whitelist_sample.txt new file mode 100644 index 00000000..e4bccdf8 --- /dev/null +++ b/whitelist_sample.txt @@ -0,0 +1,4 @@ +# Each line is a (checkid:item) tuple + +# Example: Will not consider a myignoredbucket failures as full failure. (Still printed as a warning) +check26:myignoredbucket From bf72025b9b606bf5ec4c2115daab469de2bf0090 Mon Sep 17 00:00:00 2001 From: Urjit Singh Bhatia Date: Tue, 14 Apr 2020 17:29:36 -0700 Subject: [PATCH 2/5] Ignore inline whitelist comments, pass checkid to filter ignores specifically for checks --- include/outputs | 5 +++++ prowler | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/outputs b/include/outputs index c89350b5..3f4a44c7 100644 --- a/include/outputs +++ b/include/outputs @@ -107,7 +107,12 @@ textFail(){ ## ignore whitelists for current check level="FAIL" for i in $IGNORES; do + ignore_check_name="${i%:*}" ignore_value="${i#*${CHECK_NAME}:}" + if [[ ${ignore_check_name} != "${CHECK_NAME}" ]]; then + # not for this check + continue + fi if [[ $1 =~ ${ignore_value} ]]; then level="WARNING" break diff --git a/prowler b/prowler index 05ee1e00..4e027eb2 100755 --- a/prowler +++ b/prowler @@ -218,7 +218,8 @@ REGIONS=$($AWSCLI ec2 describe-regions --query 'Regions[].RegionName' \ # Pre-process whitelist file if supplied if [[ -n "$WHITELIST_FILE" ]]; then # ignore lines starting with # (comments) - WHITELIST=$(awk '!/^[[:space:]]*#/{print }' <(cat "$WHITELIST_FILE")) + # ignore inline comments: check1:foo # inline comment + WHITELIST=$(awk '!/^[[:space:]]*#/{print }' <(cat "$WHITELIST_FILE") | sed 's/[[:space:]]*#.*$//g') fi # Load all of the groups of checks inside groups folder named as "groupNumber*" @@ -288,7 +289,7 @@ execute_check() { show_check_title "$1" ignores=$(awk '/${1}/{print}' <(echo "${WHITELIST}")) # set the custom ignores list for this check - IGNORES="${ignores}" $1 + IGNORES="${ignores}" CHECK_NAME="$1" $1 else textFail "ERROR! Use a valid check name (i.e. check41 or extra71)"; exit $EXITCODE From 103782f72b6bd8cd8cc93f758e73aa7f812464a9 Mon Sep 17 00:00:00 2001 From: Urjit Singh Bhatia Date: Mon, 4 May 2020 14:37:30 -0700 Subject: [PATCH 3/5] Fix warning handling with changes to official master --- include/colors | 2 +- prowler | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/colors b/include/colors index 7bb9f84e..0c9a4435 100644 --- a/include/colors +++ b/include/colors @@ -68,6 +68,6 @@ fi printColorsCode(){ if [[ $MONOCHROME -eq 0 ]]; then echo -e "\n$NORMAL Colors code for results: " - echo -e "$NOTICE INFO (Information)$NORMAL,$OK PASS (Recommended value)$NORMAL, $BAD FAIL (Fix required)$NORMAL, $PURPLE Not Scored $NORMAL" + echo -e "$NOTICE INFO (Information)$NORMAL,$OK PASS (Recommended value)$NORMAL, $WARNING WARNING (Ignored by whitelist)$NORMAL, $BAD FAIL (Fix required)$NORMAL, $PURPLE Not Scored $NORMAL" fi } diff --git a/prowler b/prowler index 4a8d9d4f..eb1f1c53 100755 --- a/prowler +++ b/prowler @@ -234,7 +234,7 @@ REGIONS=$($AWSCLI ec2 describe-regions --query 'Regions[].RegionName' \ if [[ -n "$WHITELIST_FILE" ]]; then # ignore lines starting with # (comments) # ignore inline comments: check1:foo # inline comment - WHITELIST=$(awk '!/^[[:space:]]*#/{print }' <(cat "$WHITELIST_FILE") | sed 's/[[:space:]]*#.*$//g') + WHITELIST="$(awk '!/^[[:space:]]*#/{print }' <(cat "$WHITELIST_FILE") | sed 's/[[:space:]]*#.*$//g')" fi # Load all of the groups of checks inside groups folder named as "groupNumber*" @@ -313,6 +313,9 @@ execute_check() { ASFF_RESOURCE_TYPE="${!asff_resource_type_var:-AwsAccount}" # Generate the credential report, only if it is group1 related which checks we # run so that the checks can safely assume it's available + # set the custom ignores list for this check + ignores="$(awk "/${1}/{print}" <(echo "${WHITELIST}"))" + if [ ${alternate_name} ];then if [[ ${alternate_name} == check1* || ${alternate_name} == extra71 ]];then if [ ! -s $TEMP_REPORT_FILE ];then @@ -325,7 +328,7 @@ execute_check() { prepare_junit_check_output "$1" fi # Execute the check - ${alternate_name} + IGNORES="${ignores}" CHECK_NAME="$1" ${alternate_name} if is_junit_output_enabled; then finalise_junit_check_output "$1" fi @@ -341,8 +344,6 @@ execute_check() { fi fi show_check_title "$1" - # set the custom ignores list for this check - ignores=$(awk '/${1}/{print}' <(echo "${WHITELIST}")) if is_junit_output_enabled; then prepare_junit_check_output "$1" fi From 5ac9be329282526ee897b00479dbe6307bcc3b09 Mon Sep 17 00:00:00 2001 From: Urjit Singh Bhatia Date: Mon, 4 May 2020 14:48:04 -0700 Subject: [PATCH 4/5] correct color info line for warning --- include/colors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/colors b/include/colors index 0c9a4435..b3f27b23 100644 --- a/include/colors +++ b/include/colors @@ -68,6 +68,6 @@ fi printColorsCode(){ if [[ $MONOCHROME -eq 0 ]]; then echo -e "\n$NORMAL Colors code for results: " - echo -e "$NOTICE INFO (Information)$NORMAL,$OK PASS (Recommended value)$NORMAL, $WARNING WARNING (Ignored by whitelist)$NORMAL, $BAD FAIL (Fix required)$NORMAL, $PURPLE Not Scored $NORMAL" + echo -e "$NOTICE INFO (Information)$NORMAL,$OK PASS (Recommended value)$NORMAL, $BAD WARNING (Ignored by whitelist)$NORMAL, $BAD FAIL (Fix required)$NORMAL, $PURPLE Not Scored $NORMAL" fi } From 8cdf3838a06812a0e807c91bffd82d95d50b3c5d Mon Sep 17 00:00:00 2001 From: Urjit Singh Bhatia Date: Mon, 4 May 2020 16:33:50 -0700 Subject: [PATCH 5/5] Print warnings with the right color code --- include/colors | 4 ++-- include/outputs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/colors b/include/colors index b3f27b23..dd8cf233 100644 --- a/include/colors +++ b/include/colors @@ -47,7 +47,7 @@ else # Colors # NOTE: Your editor may NOT show the 0x1b / escape character left of the '[' NORMAL="" - WARNING="" # Bad (red) + WARNING="" # Warning (brown) SECTION="" # Section (yellow) NOTICE="" # Notice (yellow) OK="" # Ok (green) @@ -68,6 +68,6 @@ fi printColorsCode(){ if [[ $MONOCHROME -eq 0 ]]; then echo -e "\n$NORMAL Colors code for results: " - echo -e "$NOTICE INFO (Information)$NORMAL,$OK PASS (Recommended value)$NORMAL, $BAD WARNING (Ignored by whitelist)$NORMAL, $BAD FAIL (Fix required)$NORMAL, $PURPLE Not Scored $NORMAL" + echo -e "$NOTICE INFO (Information)$NORMAL,$OK PASS (Recommended value)$NORMAL, $WARNING WARNING (Ignored by whitelist)$NORMAL, $BAD FAIL (Fix required)$NORMAL, $PURPLE Not Scored $NORMAL" fi } diff --git a/include/outputs b/include/outputs index 8931fc27..77ed7d3c 100644 --- a/include/outputs +++ b/include/outputs @@ -86,6 +86,7 @@ textInfo(){ textFail(){ ## ignore whitelists for current check level="FAIL" + colorcode="$BAD" for i in $IGNORES; do ignore_check_name="${i%:*}" ignore_value="${i#*${CHECK_NAME}:}" @@ -95,6 +96,7 @@ textFail(){ fi if [[ $1 =~ ${ignore_value} ]]; then level="WARNING" + colorcode="$WARNING" break fi done @@ -128,10 +130,10 @@ textFail(){ output_junit_failure "$1" fi if [[ "${MODES[@]}" =~ "mono" ]]; then - echo " $BAD ${level}! $1 $NORMAL" | tee -a ${OUTPUT_FILE_NAME}.$EXTENSION_TEXT + echo " $colorcode ${level}! $1 $NORMAL" | tee -a ${OUTPUT_FILE_NAME}.$EXTENSION_TEXT fi if [[ "${MODES[@]}" =~ "text" ]]; then - echo " $BAD ${level}! $1 $NORMAL" + echo " $colorcode ${level}! $1 $NORMAL" fi }