Merge branch 'master' into improve-listing-of-checks-and-groups-545

This commit is contained in:
Marc Jay
2020-04-20 18:11:06 +01:00
9 changed files with 332 additions and 122 deletions

View File

@@ -14,16 +14,15 @@
IFS=',' read -ra MODES <<< "${MODE}"
for MODE in "${MODES[@]}"; do
if [[ "$MODE" != "mono" && "$MODE" != "text" && "$MODE" != "csv" && "$MODE" != "json" && "$MODE" != "json-asff" ]]; then
echo -e "${OPTRED}ERROR!$OPTNORMAL Invalid output mode. Choose text, mono, csv, json or json-asff. ./prowler -h for help"
if [[ "$MODE" != "mono" && "$MODE" != "text" && "$MODE" != "csv" && "$MODE" != "json" && "$MODE" != "json-asff" && "$MODE" != "junit-xml" ]]; then
echo -e "${OPTRED}ERROR!$OPTNORMAL Invalid output mode. Choose text, mono, csv, json, json-asff or junit-xml. ./prowler -h for help"
EXITCODE=1
exit $EXITCODE
fi
done
if [[ "$MODE" == "mono" || "$MODE" == "csv" || "$MODE" == "json" || "$MODE" == "json-asff" ]]; then
MONOCHROME=1
fi
if [[ "$MODE" == "mono" || "$MODE" == "csv" || "$MODE" == "json" || "$MODE" == "json-asff" ]]; then
MONOCHROME=1
fi
done
if [[ $MONOCHROME -eq 1 ]]; then
# Colors

97
include/junit_integration Normal file
View File

@@ -0,0 +1,97 @@
#!/usr/bin/env bash
# Prowler - the handy cloud security tool (copyright 2018) by Toni de la Fuente
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
# Generates JUnit XML reports which can be read by Jenkins or other CI tools
JUNIT_OUTPUT_DIRECTORY="junit-reports"
is_junit_output_enabled() {
if [[ ${MODES[@]} =~ "junit-xml" ]]; then
true
else
false
fi
}
xml_escape() {
sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/\"/\&quot;/g; s/'"'"'/\&#39;/g' <<< "$1"
}
prepare_junit_output() {
# Remove any JUnit output from previous runs
rm -rf "$JUNIT_OUTPUT_DIRECTORY"
mkdir "$JUNIT_OUTPUT_DIRECTORY"
echo ""
echo "$NOTICE Writing JUnit XML reports to $PROWLER_DIR/$JUNIT_OUTPUT_DIRECTORY $NORMAL"
}
prepare_junit_check_output() {
# JUnit test cases must be named uniquely, but each Prowler check can output many times due to multiple resources,
# therefore append an index value to the test case name to provide uniqueness, reset it to 1 before starting this check
JUNIT_CHECK_INDEX=1
# To match JUnit behaviour in Java, and ensure that an aborted execution does not leave a partially written and therefore invalid XML file,
# output a JUnit XML file per check
JUNIT_OUTPUT_FILE="$JUNIT_OUTPUT_DIRECTORY/TEST-$1.xml"
printf '%s\n' \
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>" \
"<testsuite name=\"$(xml_escape "$(get_junit_classname)")\" timestamp=\"$(get_iso8601_timestamp)\">" \
" <properties>" \
" <property name=\"prowler.version\" value=\"$(xml_escape "$PROWLER_VERSION")\"/>" \
" <property name=\"aws.profile\" value=\"$(xml_escape "$PROFILE")\"/>" \
" <property name=\"aws.accountNumber\" value=\"$(xml_escape "$ACCOUNT_NUM")\"/>" \
" <property name=\"check.id\" value=\"$(xml_escape "$TITLE_ID")\"/>" \
" <property name=\"check.scored\" value=\"$(xml_escape "$ITEM_SCORED")\"/>" \
" <property name=\"check.level\" value=\"$(xml_escape "$ITEM_LEVEL")\"/>" \
" <property name=\"check.asff.type\" value=\"$(xml_escape "$ASFF_TYPE")\"/>" \
" <property name=\"check.asff.resourceType\" value=\"$(xml_escape "$ASFF_RESOURCE_TYPE")\"/>" \
" </properties>" \
> "$JUNIT_OUTPUT_FILE"
JUNIT_CHECK_START_TIME=$(get_time_in_milliseconds)
}
finalise_junit_check_output() {
echo '</testsuite>' >> "$JUNIT_OUTPUT_FILE"
}
output_junit_success() {
output_junit_test_case "$1" "<system-out>$(xml_escape "$1")</system-out>"
}
output_junit_info() {
# Nothing to output for JUnit for this level of message, but reset the check timer for timing the next check
JUNIT_CHECK_START_TIME=$(get_time_in_milliseconds)
}
output_junit_failure() {
output_junit_test_case "$1" "<failure message=\"$(xml_escape "$1")\"/>"
}
get_junit_classname() {
# <section>.<check_id> naturally follows a Java package structure, so it is suitable as a package name
echo "$TITLE_ID"
}
output_junit_test_case() {
local time_now
local test_case_duration
time_now=$(get_time_in_milliseconds)
# JUnit test case time values are in seconds, so divide by 1000 using e-3 to convert from milliseconds without losing accuracy due to non-floating point arithmetic
test_case_duration=$(printf "%.3f" "$((time_now - JUNIT_CHECK_START_TIME))e-3")
printf '%s\n' \
" <testcase name=\"$(xml_escape "$TITLE_TEXT") ($JUNIT_CHECK_INDEX)\" classname=\"$(xml_escape "$(get_junit_classname)")\" time=\"$test_case_duration\">" \
" $2" \
" </testcase>" >> "$JUNIT_OUTPUT_FILE"
# Reset the check timer for timing the next check
JUNIT_CHECK_START_TIME=$(get_time_in_milliseconds)
((JUNIT_CHECK_INDEX+=1))
}

View File

@@ -11,17 +11,19 @@
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
DATE_CMD="date"
gnu_how_older_from_today() {
DATE_TO_COMPARE=$1
TODAY_IN_DAYS=$(date -d "$(date +%Y-%m-%d)" +%s)
DATE_FROM_IN_DAYS=$(date -d $DATE_TO_COMPARE +%s)
TODAY_IN_DAYS=$("$DATE_CMD" -d "$("$DATE_CMD" +%Y-%m-%d)" +%s)
DATE_FROM_IN_DAYS=$("$DATE_CMD" -d $DATE_TO_COMPARE +%s)
DAYS_SINCE=$((($TODAY_IN_DAYS - $DATE_FROM_IN_DAYS )/60/60/24))
echo $DAYS_SINCE
}
bsd_how_older_from_today() {
DATE_TO_COMPARE=$1
TODAY_IN_DAYS=$(date +%s)
DATE_FROM_IN_DAYS=$(date -jf %Y-%m-%d $DATE_TO_COMPARE +%s)
TODAY_IN_DAYS=$("$DATE_CMD" +%s)
DATE_FROM_IN_DAYS=$("$DATE_CMD" -jf %Y-%m-%d $DATE_TO_COMPARE +%s)
DAYS_SINCE=$((($TODAY_IN_DAYS - $DATE_FROM_IN_DAYS )/60/60/24))
echo $DAYS_SINCE
}
@@ -31,13 +33,13 @@ bsd_how_older_from_today() {
gnu_timestamp_to_date() {
# remove fractions of a second
TIMESTAMP_TO_CONVERT=$(echo $1 | cut -f1 -d".")
OUTPUT_DATE=$(date -d @$TIMESTAMP_TO_CONVERT +'%Y-%m-%d')
OUTPUT_DATE=$("$DATE_CMD" -d @$TIMESTAMP_TO_CONVERT +'%Y-%m-%d')
echo $OUTPUT_DATE
}
bsd_timestamp_to_date() {
# remove fractions of a second
TIMESTAMP_TO_CONVERT=$(echo $1 | cut -f1 -d".")
OUTPUT_DATE=$(date -r $TIMESTAMP_TO_CONVERT +'%Y-%m-%d')
OUTPUT_DATE=$("$DATE_CMD" -r $TIMESTAMP_TO_CONVERT +'%Y-%m-%d')
echo $OUTPUT_DATE
}
@@ -50,15 +52,15 @@ bsd_decode_report() {
gnu_how_many_days_from_today() {
DATE_TO_COMPARE=$1
TODAY_IN_DAYS=$(date -d "$(date +%Y-%m-%d)" +%s)
DATE_IN_DAYS=$(date -d $DATE_TO_COMPARE +%s)
TODAY_IN_DAYS=$("$DATE_CMD" -d "$("$DATE_CMD" +%Y-%m-%d)" +%s)
DATE_IN_DAYS=$("$DATE_CMD" -d $DATE_TO_COMPARE +%s)
DAYS_TO=$((( $DATE_IN_DAYS - $TODAY_IN_DAYS )/60/60/24))
echo $DAYS_TO
}
bsd_how_many_days_from_today() {
DATE_TO_COMPARE=$1
TODAY_IN_DAYS=$(date +%s)
DATE_IN_DAYS=$(date -jf %Y-%m-%d $DATE_TO_COMPARE +%s)
TODAY_IN_DAYS=$("$DATE_CMD" +%s)
DATE_IN_DAYS=$("$DATE_CMD" -jf %Y-%m-%d $DATE_TO_COMPARE +%s)
DAYS_TO=$((( $DATE_IN_DAYS - $TODAY_IN_DAYS )/60/60/24))
echo $DAYS_TO
}
@@ -66,17 +68,32 @@ bsd_how_many_days_from_today() {
gnu_get_date_previous_than_months() {
MONTHS_TO_COMPARE=$1
MONTHS_TO_COMPARE_IN_SECONDS=$(( 60 * 60 * 24 * 31 * $MONTHS_TO_COMPARE ))
CURRENTSECS=$(date +%s)
CURRENTSECS=$("$DATE_CMD" +%s)
STARTDATEINSECS=$(( $CURRENTSECS - $MONTHS_TO_COMPARE_IN_SECONDS ))
DATE_BEFORE_MONTHS_TO_COMPARE=$(date -d @$STARTDATEINSECS '+%Y-%m-%d')
DATE_BEFORE_MONTHS_TO_COMPARE=$("$DATE_CMD" -d @$STARTDATEINSECS '+%Y-%m-%d')
echo $DATE_BEFORE_MONTHS_TO_COMPARE
}
bsd_get_date_previous_than_months() {
MONTHS_TO_COMPARE=$1
DATE_BEFORE_MONTHS_TO_COMPARE=$(date -v -$(echo $MONTHS_TO_COMPARE)m '+%Y-%m-%d')
DATE_BEFORE_MONTHS_TO_COMPARE=$("$DATE_CMD" -v -$(echo $MONTHS_TO_COMPARE)m '+%Y-%m-%d')
echo $DATE_BEFORE_MONTHS_TO_COMPARE
}
gnu_get_time_in_milliseconds() {
"$DATE_CMD" +%s%3N
}
bsd_get_time_in_milliseconds() {
# BSD date does not support outputting milliseconds, so pad with zeros
"$DATE_CMD" +%s000
}
gnu_get_iso8601_timestamp() {
"$DATE_CMD" -u +"%Y-%m-%dT%H:%M:%SZ"
}
bsd_get_iso8601_timestamp() {
"$DATE_CMD" -u +"%Y-%m-%dT%H:%M:%SZ"
}
gnu_test_tcp_connectivity() {
HOST=$1
PORT=$2
@@ -114,16 +131,28 @@ if [ "$OSTYPE" == "linux-gnu" ] || [ "$OSTYPE" == "linux-musl" ]; then
get_date_previous_than_months() {
gnu_get_date_previous_than_months "$1"
}
get_time_in_milliseconds() {
gnu_get_time_in_milliseconds
}
get_iso8601_timestamp() {
gnu_get_iso8601_timestamp
}
test_tcp_connectivity() {
gnu_test_tcp_connectivity "$1" "$2" "$3"
}
elif [[ "$OSTYPE" == "darwin"* ]]; then
# BSD/OSX commands compatibility
TEMP_REPORT_FILE=$(mktemp -t prowler.cred_report-XXXXXX)
# It is possible that the user has installed GNU coreutils, replacing the default Mac OS X BSD tools with
# GNU coreutils equivalents. Only GNU date allows --version as a valid argument, so use the validity of this argument
# It is possible that the user has installed GNU coreutils on OS X. By default, this will make GNU commands
# available with a 'g' prefix, e.g. 'gdate'. Test if this is present, and use it if so, as it supports more features.
# The user also may have replaced the default Mac OS X BSD tools with the GNU coreutils equivalents.
# Only GNU date allows --version as a valid argument, so use the validity of this argument
# as a means to detect that coreutils is installed and is overriding the default tools
if date --version >/dev/null 2>&1 ; then
GDATE=$(which gdate)
if [ -n "${GDATE}" ]; then
DATE_CMD="gdate"
fi
if "$DATE_CMD" --version >/dev/null 2>&1 ; then
how_older_from_today() {
gnu_how_older_from_today "$1"
}
@@ -139,6 +168,12 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
get_date_previous_than_months() {
gnu_get_date_previous_than_months "$1"
}
get_time_in_milliseconds() {
gnu_get_time_in_milliseconds
}
get_iso8601_timestamp() {
gnu_get_iso8601_timestamp
}
else
how_older_from_today() {
bsd_how_older_from_today "$1"
@@ -155,6 +190,12 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
get_date_previous_than_months() {
bsd_get_date_previous_than_months "$1"
}
get_time_in_milliseconds() {
bsd_get_time_in_milliseconds
}
get_iso8601_timestamp() {
bsd_get_iso8601_timestamp
}
fi
test_tcp_connectivity() {
bsd_test_tcp_connectivity "$1" "$2" "$3"
@@ -177,6 +218,12 @@ elif [[ "$OSTYPE" == "cygwin" ]]; then
get_date_previous_than_months() {
gnu_get_date_previous_than_months "$1"
}
get_time_in_milliseconds() {
gnu_get_time_in_milliseconds
}
get_iso8601_timestamp() {
gnu_get_iso8601_timestamp
}
test_tcp_connectivity() {
gnu_test_tcp_connectivity "$1" "$2" "$3"
}

View File

@@ -27,26 +27,28 @@ textPass(){
fi
PASS_COUNTER=$((PASS_COUNTER+1))
if [[ "${MODES[@]}" =~ "csv" || "${MODES[@]}" =~ "json" || "${MODES[@]}" =~ "json-asff" ]]; then
if [[ $2 ]]; then
REPREGION=$2
else
REPREGION=$REGION
fi
if [[ "${MODES[@]}" =~ "csv" ]]; then
echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}PASS${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV
fi
if [[ "${MODES[@]}" =~ "json" ]]; then
generateJsonOutput "$1" "Pass" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON
fi
if [[ "${MODES[@]}" =~ "json-asff" ]]; then
JSON_ASFF_OUTPUT=$(generateJsonAsffOutput "$1" "PASSED" "INFORMATIONAL")
echo "${JSON_ASFF_OUTPUT}" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_ASFF
if [[ "${SEND_TO_SECURITY_HUB}" -eq 1 ]]; then
sendToSecurityHub "${JSON_ASFF_OUTPUT}"
fi
fi
if [[ $2 ]]; then
REPREGION=$2
else
REPREGION=$REGION
fi
if [[ "${MODES[@]}" =~ "csv" ]]; then
echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}PASS${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV
fi
if [[ "${MODES[@]}" =~ "json" ]]; then
generateJsonOutput "$1" "Pass" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON
fi
if [[ "${MODES[@]}" =~ "json-asff" ]]; then
JSON_ASFF_OUTPUT=$(generateJsonAsffOutput "$1" "PASSED" "INFORMATIONAL")
echo "${JSON_ASFF_OUTPUT}" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_ASFF
if [[ "${SEND_TO_SECURITY_HUB}" -eq 1 ]]; then
sendToSecurityHub "${JSON_ASFF_OUTPUT}"
fi
fi
if is_junit_output_enabled; then
output_junit_success "$1"
fi
if [[ "${MODES[@]}" =~ "text" ]]; then
echo " $OK PASS!$NORMAL $1"
fi
}
@@ -56,19 +58,21 @@ textInfo(){
return
fi
if [[ "${MODES[@]}" =~ "csv" || "${MODES[@]}" =~ "json" || "${MODES[@]}" =~ "json-asff" ]]; then
if [[ $2 ]]; then
REPREGION=$2
else
REPREGION=$REGION
fi
if [[ "${MODES[@]}" =~ "csv" ]]; then
echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}INFO${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV
fi
if [[ "${MODES[@]}" =~ "json" ]]; then
generateJsonOutput "$1" "Info" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON
fi
if [[ $2 ]]; then
REPREGION=$2
else
REPREGION=$REGION
fi
if [[ "${MODES[@]}" =~ "csv" ]]; then
echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}INFO${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV
fi
if [[ "${MODES[@]}" =~ "json" ]]; then
generateJsonOutput "$1" "Info" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON
fi
if is_junit_output_enabled; then
output_junit_info "$1"
fi
if [[ "${MODES[@]}" =~ "text" ]]; then
echo " $NOTICE INFO! $1 $NORMAL"
fi
}
@@ -76,26 +80,28 @@ textInfo(){
textFail(){
FAIL_COUNTER=$((FAIL_COUNTER+1))
EXITCODE=3
if [[ "${MODES[@]}" =~ "csv" || "${MODES[@]}" =~ "json" || "${MODES[@]}" =~ "json-asff" ]]; then
if [[ $2 ]]; then
REPREGION=$2
else
REPREGION=$REGION
fi
if [[ "${MODES[@]}" =~ "csv" ]]; then
echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}FAIL${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV
fi
if [[ "${MODES[@]}" =~ "json" ]]; then
generateJsonOutput "$1" "Fail" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON
fi
if [[ "${MODES[@]}" =~ "json-asff" ]]; then
JSON_ASFF_OUTPUT=$(generateJsonAsffOutput "$1" "FAILED" "HIGH")
echo "${JSON_ASFF_OUTPUT}" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_ASFF
if [[ "${SEND_TO_SECURITY_HUB}" -eq 1 ]]; then
sendToSecurityHub "${JSON_ASFF_OUTPUT}"
fi
fi
if [[ $2 ]]; then
REPREGION=$2
else
REPREGION=$REGION
fi
if [[ "${MODES[@]}" =~ "csv" ]]; then
echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}FAIL${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV
fi
if [[ "${MODES[@]}" =~ "json" ]]; then
generateJsonOutput "$1" "Fail" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON
fi
if [[ "${MODES[@]}" =~ "json-asff" ]]; then
JSON_ASFF_OUTPUT=$(generateJsonAsffOutput "$1" "FAILED" "HIGH")
echo "${JSON_ASFF_OUTPUT}" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_ASFF
if [[ "${SEND_TO_SECURITY_HUB}" -eq 1 ]]; then
sendToSecurityHub "${JSON_ASFF_OUTPUT}"
fi
fi
if is_junit_output_enabled; then
output_junit_failure "$1"
fi
if [[ "${MODES[@]}" =~ "text" ]]; then
echo " $BAD FAIL! $1 $NORMAL"
fi
}
@@ -161,7 +167,7 @@ generateJsonOutput(){
--arg ITEM_LEVEL "$ITEM_LEVEL" \
--arg TITLE_ID "$TITLE_ID" \
--arg REPREGION "$REPREGION" \
--arg TIMESTAMP $(date -u +"%Y-%m-%dT%H:%M:%SZ") \
--arg TIMESTAMP "$(get_iso8601_timestamp)" \
-n '{
"Profile": $PROFILE,
"Account Number": $ACCOUNT_NUM,
@@ -183,20 +189,17 @@ generateJsonAsffOutput(){
local status=$2
local severity=$3
jq -M -c \
--arg PROFILE "$PROFILE" \
--arg ACCOUNT_NUM "$ACCOUNT_NUM" \
--arg TITLE_TEXT "$TITLE_TEXT" \
--arg MESSAGE "$(echo -e "${message}" | sed -e 's/^[[:space:]]*//')" \
--arg UNIQUE_ID "$(LC_ALL=C echo -e "${message}" | tr -cs '[:alnum:]._~-\n' '_')" \
--arg STATUS "$status" \
--arg SEVERITY "$severity" \
--arg SCORED "$ITEM_SCORED" \
--arg ITEM_LEVEL "$ITEM_LEVEL" \
--arg TITLE_ID "$TITLE_ID" \
--arg TYPE "$ASFF_TYPE" \
--arg RESOURCE_TYPE "$ASFF_RESOURCE_TYPE" \
--arg REPREGION "$REPREGION" \
--arg TIMESTAMP $(date -u +"%Y-%m-%dT%H:%M:%SZ") \
--arg TIMESTAMP "$(get_iso8601_timestamp)" \
--arg PROWLER_VERSION "$PROWLER_VERSION" \
-n '{
"SchemaVersion": "2018-10-08",