From 92e1f17a8049f7d21b8f0d8ff8542c7d2ebf3ab3 Mon Sep 17 00:00:00 2001 From: Marc Jay Date: Tue, 7 Apr 2020 16:08:07 +0100 Subject: [PATCH] Adds 'json-asff' and 'securityhub' output modes json-asff mode outputs JSON, similar to the standard 'json' mode with one check per line, but in AWS Security Finding Format - used by AWS Security Hub Currently uses a generic Type, Resources and ProductArn value, but sets the Id to a unique value that includes the details of the message, in order to separate out checks that run against multiple resources and output one result per resource per check. This ensures that findings can be updated, should the resource move in or out of compliance securityhub mode generates the ASFF JSON and then passes it to an 'aws securityhub batch-import-findings' call, once per resource per check. Output to the screen is similar to the standard mode, but prints whether or not the finding was submitted successfully Fixes #524 --- checks/check13 | 6 +- include/colors | 6 +- include/outputs | 202 +++++++++++++++++++++++++++--------------------- include/whoami | 2 +- prowler | 12 +-- 5 files changed, 127 insertions(+), 101 deletions(-) diff --git a/checks/check13 b/checks/check13 index aeab1944..10289768 100644 --- a/checks/check13 +++ b/checks/check13 @@ -17,17 +17,17 @@ CHECK_ALTERNATE_check103="check13" check13(){ # "Ensure credentials unused for 90 days or greater are disabled (Scored)" COMMAND12_LIST_USERS_WITH_PASSWORD_ENABLED=$(cat $TEMP_REPORT_FILE|awk -F, '{ print $1,$4 }' |grep true | awk '{ print $1 }') - # Only check Password last used for users with password enabled + # Only check Password last used for users with password enabled if [[ $COMMAND12_LIST_USERS_WITH_PASSWORD_ENABLED ]]; then for i in $COMMAND12_LIST_USERS_WITH_PASSWORD_ENABLED; do DATEUSED=$($AWSCLI iam list-users --query "Users[?UserName=='$i'].PasswordLastUsed" --output text $PROFILE_OPT --region $REGION | cut -d'T' -f1) if [ "$DATEUSED" == "" ] then - textFail "User \"$i\" has not logged in during the last 90 days " + textFail "User \"$i\" has not logged in during the last 90 days" else HOWOLDER=$(how_older_from_today $DATEUSED) if [ $HOWOLDER -gt "90" ];then - textFail "User \"$i\" has not logged in during the last 90 days " + textFail "User \"$i\" has not logged in during the last 90 days" else textPass "User \"$i\" found with credentials used in the last 90 days" fi diff --git a/include/colors b/include/colors index 2b7175ce..68ac32a4 100644 --- a/include/colors +++ b/include/colors @@ -11,15 +11,15 @@ # CONDITIONS OF ANY KIND, either express or implied. See the License for the # specific language governing permissions and limitations under the License. -if [[ $MODE != "mono" && $MODE != "text" && $MODE != "csv" && $MODE != "json" ]]; then +if [[ "$MODE" != "mono" && "$MODE" != "text" && "$MODE" != "csv" && "$MODE" != "json" && "$MODE" != "json-asff" && "$MODE" != "securityhub" ]]; then echo "" - echo "$OPTRED ERROR!$OPTNORMAL Invalid output mode. Choose text, mono, or csv." + echo "$OPTRED ERROR!$OPTNORMAL Invalid output mode. Choose text, mono, csv, json, json-asff or securityhub." usage EXITCODE=1 exit $EXITCODE fi -if [[ "$MODE" == "mono" || "$MODE" == "csv" || "$MODE" == "json" ]]; then +if [[ "$MODE" == "mono" || "$MODE" == "csv" || "$MODE" == "json" || "$MODE" == "json-asff" ]]; then MONOCHROME=1 fi diff --git a/include/outputs b/include/outputs index b18ac683..6af14778 100644 --- a/include/outputs +++ b/include/outputs @@ -18,41 +18,22 @@ textPass(){ fi PASS_COUNTER=$((PASS_COUNTER+1)) - if [[ "$MODE" == "csv" ]]; then + if [[ "$MODE" == "csv" || "$MODE" == "json" || "$MODE" == "json-asff" || "$MODE" == "securityhub" ]]; then if [[ $2 ]]; then REPREGION=$2 else REPREGION=$REGION fi - echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}PASS${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" - elif [[ "$MODE" == "json" ]]; then - if [[ $2 ]]; then - REPREGION=$2 - else - REPREGION=$REGION + if [[ "$MODE" == "csv" ]]; then + echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}PASS${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" + elif [[ "$MODE" == "json" ]]; then + generateJsonOutput "$1" "Pass" + elif [[ "$MODE" == "json-asff" ]]; then + generateJsonAsffOutput "$1" "PASSED" "INFORMATIONAL" + elif [[ "$MODE" == "securityhub" ]]; then + printf " $OK PASS!$NORMAL %s... " "$1" + aws securityhub batch-import-findings --findings "$(generateJsonAsffOutput "$1" "PASSED" "INFORMATIONAL")" | jq -M -r 'if .SuccessCount == 1 then "Successfully submitted finding" else "Failed to upload finding" end' fi - jq -M -c \ - --arg PROFILE "$PROFILE" \ - --arg ACCOUNT_NUM "$ACCOUNT_NUM" \ - --arg TITLE_TEXT "$TITLE_TEXT" \ - --arg MESSAGE "$(echo -e "${1}" | sed -e 's/^[[:space:]]*//')" \ - --arg SCORED "$ITEM_SCORED" \ - --arg ITEM_LEVEL "$ITEM_LEVEL" \ - --arg TITLE_ID "$TITLE_ID" \ - --arg REPREGION "$REPREGION" \ - --arg TIMESTAMP $(date -u +"%Y-%m-%dT%H:%M:%SZ") \ - -n '{ - "Profile": $PROFILE, - "Account Number": $ACCOUNT_NUM, - "Control": $TITLE_TEXT, - "Message": $MESSAGE, - "Status": "Pass", - "Scored": $SCORED, - "Level": $ITEM_LEVEL, - "Control ID": $TITLE_ID, - "Region": $REPREGION, - "Timestamp": $TIMESTAMP, - }' else echo " $OK PASS!$NORMAL $1" fi @@ -63,41 +44,22 @@ textInfo(){ return fi - if [[ "$MODE" == "csv" ]]; then + if [[ "$MODE" == "csv" || "$MODE" == "json" || "$MODE" == "json-asff" ]]; then if [[ $2 ]]; then REPREGION=$2 else REPREGION=$REGION fi - echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}INFO${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" - elif [[ "$MODE" == "json" ]]; then - if [[ $2 ]]; then - REPREGION=$2 - else - REPREGION=$REGION + if [[ "$MODE" == "csv" ]]; then + echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}INFO${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" + elif [[ "$MODE" == "json" ]]; then + generateJsonOutput "$1" "Info" + elif [[ "$MODE" == "json-asff" ]]; then + generateJsonAsffOutput "$1" "NOT_AVAILABLE" "LOW" + elif [[ "$MODE" == "securityhub" ]]; then + printf " $NOTICE INFO! %s... $NORMAL" "$1" + aws securityhub batch-import-findings --findings "$(generateJsonAsffOutput "$1" "NOT_AVAILABLE" "LOW")" | jq -M -r 'if .SuccessCount == 1 then "Successfully submitted finding" else "Failed to upload finding" end' fi - jq -M -c \ - --arg PROFILE "$PROFILE" \ - --arg ACCOUNT_NUM "$ACCOUNT_NUM" \ - --arg TITLE_TEXT "$TITLE_TEXT" \ - --arg MESSAGE "$(echo -e "${1}" | sed -e 's/^[[:space:]]*//')" \ - --arg SCORED "$ITEM_SCORED" \ - --arg ITEM_LEVEL "$ITEM_LEVEL" \ - --arg TITLE_ID "$TITLE_ID" \ - --arg REPREGION "$REPREGION" \ - --arg TIMESTAMP $(date -u +"%Y-%m-%dT%H:%M:%SZ") \ - -n '{ - "Profile": $PROFILE, - "Account Number": $ACCOUNT_NUM, - "Control": $TITLE_TEXT, - "Message": $MESSAGE, - "Status": "Info", - "Scored": $SCORED, - "Level": $ITEM_LEVEL, - "Control ID": $TITLE_ID, - "Region": $REPREGION, - "Timestamp": $TIMESTAMP, - }' else echo " $NOTICE INFO! $1 $NORMAL" fi @@ -106,41 +68,22 @@ textInfo(){ textFail(){ FAIL_COUNTER=$((FAIL_COUNTER+1)) EXITCODE=3 - if [[ "$MODE" == "csv" ]]; then + if [[ "$MODE" == "csv" || "$MODE" == "json" || "$MODE" == "json-asff" || "$MODE" == "securityhub" ]]; then if [[ $2 ]]; then REPREGION=$2 else REPREGION=$REGION fi - echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}FAIL${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" - elif [[ "$MODE" == "json" ]]; then - if [[ $2 ]]; then - REPREGION=$2 - else - REPREGION=$REGION + if [[ "$MODE" == "csv" ]]; then + echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}FAIL${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" + elif [[ "$MODE" == "json" ]]; then + generateJsonOutput "$1" "Fail" + elif [[ "$MODE" == "json-asff" ]]; then + generateJsonAsffOutput "$1" "FAILED" "HIGH" + elif [[ "$MODE" == "securityhub" ]]; then + printf " $BAD FAIL! %s... $NORMAL" "$1" + aws securityhub batch-import-findings --findings "$(generateJsonAsffOutput "$1" "FAILED" "HIGH")" | jq -M -r 'if .SuccessCount == 1 then "Successfully submitted finding" else "Failed to upload finding" end' fi - jq -M -c \ - --arg PROFILE "$PROFILE" \ - --arg ACCOUNT_NUM "$ACCOUNT_NUM" \ - --arg TITLE_TEXT "$TITLE_TEXT" \ - --arg MESSAGE "$(echo -e "${1}" | sed -e 's/^[[:space:]]*//')" \ - --arg SCORED "$ITEM_SCORED" \ - --arg ITEM_LEVEL "$ITEM_LEVEL" \ - --arg TITLE_ID "$TITLE_ID" \ - --arg REPREGION "$REPREGION" \ - --arg TIMESTAMP $(date -u +"%Y-%m-%dT%H:%M:%SZ") \ - -n '{ - "Profile": $PROFILE, - "Account Number": $ACCOUNT_NUM, - "Control": $TITLE_TEXT, - "Message": $MESSAGE, - "Status": "Fail", - "Scored": $SCORED, - "Level": $ITEM_LEVEL, - "Control ID": $TITLE_ID, - "Region": $REPREGION, - "Timestamp": $TIMESTAMP, - }' else echo " $BAD FAIL! $1 $NORMAL" fi @@ -179,7 +122,7 @@ textTitle(){ if [[ "$MODE" == "csv" ]]; then >&2 echo "$TITLE_ID $TITLE_TEXT" - elif [[ "$MODE" == "json" ]]; then + elif [[ "$MODE" == "json" || "$MODE" == "json-asff" ]]; then : else if [[ "$ITEM_SCORED" == "Scored" ]]; then @@ -189,3 +132,86 @@ textTitle(){ fi fi } + +generateJsonOutput(){ + local message=$1 + local status=$2 + jq -M -c \ + --arg PROFILE "$PROFILE" \ + --arg ACCOUNT_NUM "$ACCOUNT_NUM" \ + --arg TITLE_TEXT "$TITLE_TEXT" \ + --arg MESSAGE "$(echo -e "${message}" | sed -e 's/^[[:space:]]*//')" \ + --arg STATUS "$status" \ + --arg SCORED "$ITEM_SCORED" \ + --arg ITEM_LEVEL "$ITEM_LEVEL" \ + --arg TITLE_ID "$TITLE_ID" \ + --arg REPREGION "$REPREGION" \ + --arg TIMESTAMP $(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + -n '{ + "Profile": $PROFILE, + "Account Number": $ACCOUNT_NUM, + "Control": $TITLE_TEXT, + "Message": $MESSAGE, + "Status": $STATUS, + "Scored": $SCORED, + "Level": $ITEM_LEVEL, + "Control ID": $TITLE_ID, + "Region": $REPREGION, + "Timestamp": $TIMESTAMP, + }' +} + +generateJsonAsffOutput(){ + # UNIQUE_ID must only contain characters from the unreserved characters set defined in section 2.3 of RFC-3986 + # Replace any successive non-conforming characters with a single underscore + local message=$1 + local status=$2 + local severity=$3 + jq -M -c \ + --arg PROFILE "$PROFILE" \ + --arg ACCOUNT_NUM "$ACCOUNT_NUM" \ + --arg TITLE_TEXT "$TITLE_TEXT" \ + --arg MESSAGE "$(echo -e "${message}" | sed -e 's/^[[:space:]]*//')" \ + --arg UNIQUE_ID "$(LC_ALL=C echo -e "${message}" | tr -cs '[:alnum:]._~-\n' '_')" \ + --arg STATUS "$status" \ + --arg SEVERITY "$severity" \ + --arg SCORED "$ITEM_SCORED" \ + --arg ITEM_LEVEL "$ITEM_LEVEL" \ + --arg TITLE_ID "$TITLE_ID" \ + --arg REPREGION "$REPREGION" \ + --arg TIMESTAMP $(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + --arg PROWLER_VERSION "$PROWLER_VERSION" \ +-n '{ + "SchemaVersion": "2018-10-08", + "Id": "prowler-\($TITLE_ID)-\($ACCOUNT_NUM)-\($REPREGION)-\($UNIQUE_ID)", + "ProductArn": "arn:aws:securityhub:\($REPREGION):\($ACCOUNT_NUM):product/\($ACCOUNT_NUM)/default", + "ProductFields": { + "ProviderName": "Prowler", + "ProviderVersion": $PROWLER_VERSION + }, + "GeneratorId": "prowler-\($PROWLER_VERSION)", + "AwsAccountId": $ACCOUNT_NUM, + "Types": [ + "Software and Configuration Checks" + ], + "FirstObservedAt": $TIMESTAMP, + "UpdatedAt": $TIMESTAMP, + "CreatedAt": $TIMESTAMP, + "Severity": { + "Label": $SEVERITY + }, + "Title": $TITLE_TEXT, + "Description": $MESSAGE, + "Resources": [ + { + "Type": "AwsAccount", + "Id": "AWS: : : :Account:\($ACCOUNT_NUM)", + "Partition": "aws", + "Region": $REPREGION + } + ], + "Compliance": { + "Status": $STATUS + } + }' +} diff --git a/include/whoami b/include/whoami index 0fa8479a..abe59cc5 100644 --- a/include/whoami +++ b/include/whoami @@ -28,7 +28,7 @@ getWhoami(){ printCsvHeader textTitle "0.0" "Show report generation info" "NOT_SCORED" "SUPPORT" textInfo "ARN: $CALLER_ARN TIMESTAMP: $SCRIPT_START_TIME" - elif [[ "$MODE" == "json" ]]; then + elif [[ "$MODE" == "json" || "$MODE" == "json-asff" ]]; then : else echo "" diff --git a/prowler b/prowler index 2486808c..01eb9a35 100755 --- a/prowler +++ b/prowler @@ -64,7 +64,7 @@ USAGE: -f specify an AWS region to run checks against (i.e.: us-west-1) -m specify the maximum number of items to return for long-running requests (default: 100) - -M output mode: text (default), mono, json, csv (separator is ","; data is on stdout; progress on stderr) + -M output mode: text (default), mono, json, json-asff, securityhub, csv (separator is ","; data is on stdout; progress on stderr) -k keep the credential report -n show check numbers to sort easier (i.e.: 1.01 instead of 1.1) @@ -77,12 +77,12 @@ USAGE: -s show scoring report -x specify external directory with custom checks (i.e. /my/own/checks, files must start by "check") -q suppress info messages and passing test output - -A account id for the account where to assume a role, requires -R and -T + -A account id for the account where to assume a role, requires -R and -T (i.e.: 123456789012) - -R role name to assume in the account, requires -A and -T + -R role name to assume in the account, requires -A and -T (i.e.: ProwlerRole) -T session durantion given to that role credentials in seconds, default 1h (3600) recommended 12h, requires -R and -T - (i.e.: 43200) + (i.e.: 43200) -h this help " exit @@ -395,8 +395,8 @@ if [[ $PRINTGROUPSONLY == "1" ]]; then exit $EXITCODE fi -# Check that jq is installed for JSON output -if [[ $MODE == "json" ]]; then +# Check that jq is installed for JSON outputs +if [[ "$MODE" == "json" || "$MODE" == "json-asff" || "$MODE" == "securityhub" ]]; then . $PROWLER_DIR/include/jq_detector fi