From 036ae640e539afbb221fb7b1fb278fd91147a737 Mon Sep 17 00:00:00 2001 From: nalansitan Date: Tue, 14 Apr 2020 10:38:01 +0800 Subject: [PATCH 1/7] support arn:aws:s3::: on extra725 --- checks/check_extra725 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checks/check_extra725 b/checks/check_extra725 index 36dd6840..88f43ce5 100644 --- a/checks/check_extra725 +++ b/checks/check_extra725 @@ -30,7 +30,7 @@ extra725(){ if [[ $LIST_OF_TRAILS ]]; then BUCKET_ENABLED_TRAILS=() for trail in $LIST_OF_TRAILS; do - BUCKET_ENABLED_IN_TRAIL=$($AWSCLI cloudtrail get-event-selectors $PROFILE_OPT --trail-name $trail --query "EventSelectors[*].DataResources[?Type == \`AWS::S3::Object\`].Values" --output text |xargs -n1| grep -E "^arn:aws:s3:::$bucketName/\S*$|^arn:aws:s3$") + BUCKET_ENABLED_IN_TRAIL=$($AWSCLI cloudtrail get-event-selectors $PROFILE_OPT --trail-name $trail --query "EventSelectors[*].DataResources[?Type == \`AWS::S3::Object\`].Values" --output text |xargs -n1| grep -E "^arn:aws:s3:::$bucketName/\S*$|^arn:aws:s3$|^arn:aws:s3:::$") if [[ $BUCKET_ENABLED_IN_TRAIL ]]; then BUCKET_ENABLED_TRAILS+=($trail) # textPass "$regx: S3 bucket $bucketName has Object-level logging enabled in trail $trail" "$regx" From 994390351eff80fa6b35ec4419bf3ef859a06480 Mon Sep 17 00:00:00 2001 From: Marc Jay Date: Wed, 15 Apr 2020 02:36:16 +0100 Subject: [PATCH 2/7] Add the ability to generate JUnit XML reports with a -J flag If the -J flag is passed, generate JUnit XML reports for each check, in-line with how Java tools generate JUnit reports. Check section numbers equate to 'root packages', checks are second-level packages, each check equates to a testsuite (mirroring Java where each test class is a testsuite) and each pass/fail of a check equates to a testcase Time the execution of each check and include this in the report Include properties (Prowler version, check level etc.) in-line with standard JUnit files XML escape all strings for safety Detect if a user has GNU coreutils installed on Mac OS X, but not as their default, switching to using gdate for date commands if so, as it has more features, including getting dates in milliseconds Add prowler-output, junit-reports and VSCode files to .gitignore Update README to include JUnit info, address markdownlint warnings Remove unused arguments to jq in generateJsonAsffOutput Fixes #537 --- .gitignore | 11 ++++- README.md | 83 +++++++++++++++++++++++++----------- include/junit_integration | 89 +++++++++++++++++++++++++++++++++++++++ include/os_detector | 79 +++++++++++++++++++++++++++------- include/outputs | 30 +++++++------ prowler | 52 +++++++++++++++++------ 6 files changed, 277 insertions(+), 67 deletions(-) create mode 100644 include/junit_integration diff --git a/.gitignore b/.gitignore index c80f14d6..3d433f87 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,13 @@ tags [._]*.un~ # MacOs DS_Store -*.DS_Store \ No newline at end of file +*.DS_Store + +# Prowler output +prowler-output-* + +# JUnit Reports +junit-reports/ + +# VSCode files +.vscode/ diff --git a/README.md b/README.md index 8d71e75a..4b07afa6 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,6 @@ Read more about [CIS Amazon Web Services Foundations Benchmark v1.2.0 - 05-23-20 - HIPAA [hipaa] Read more [here](#hipaa-checks) - Trust Boundaries [trustboundaries] Read more [here](#trustboundaries-checks) - With Prowler you can: - get a colorful or monochrome report @@ -68,6 +67,7 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX AWS-CLI can be also installed it using "brew", "apt", "yum" or manually from , but `ansi2html` and `detect-secrets` has to be installed using `pip`. You will need to install `jq` to get more accuracy in some checks. - Make sure jq is installed (example below with "apt" but use a valid package manager for your OS): + ```sh sudo apt install jq ``` @@ -84,7 +84,9 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX ```sh aws configure ``` + or + ```sh export AWS_ACCESS_KEY_ID="ASXXXXXXX" export AWS_SECRET_ACCESS_KEY="XXXXXXXXX" @@ -110,7 +112,7 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX Use `-l` to list all available checks and group of checks (sections) - If you want to avoid installing dependences run it using Docker: + If you want to avoid installing dependencies run it using Docker: ```sh docker run -ti --rm --name prowler --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_SESSION_TOKEN toniblyx/prowler:latest @@ -127,16 +129,21 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX ```sh ./prowler -c check310 ``` + With Docker: + ```sh docker run -ti --rm --name prowler --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_SESSION_TOKEN toniblyx/prowler:latest "-c check310" ``` or multiple checks separated by comma: + ```sh ./prowler -c check310,check722 ``` + or all checks but some of them: + ```sh ./prowler -E check42,check43 ``` @@ -152,25 +159,31 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX ```sh ./prowler -g group1 # for iam related checks ``` + or exclude some checks in the group: + ```sh ./prowler -g group4 -E check42,check43 ``` Valid check numbers are based on the AWS CIS Benchmark guide, so 1.1 is check11 and 3.10 is check310 -### Save your reports +### Save your reports 1. If you want to save your report for later analysis thare are different ways, natively (supported text, mono, csv, json and json-asff see note below for more info): ```sh ./prowler -M csv ``` + or with multiple formats at the same time: + ```sh ./prowler -M csv,json,json-asff ``` + or just a group of checks in multiple formats: + ```sh ./prowler -g gdpr -M csv,json,json-asff ``` @@ -190,7 +203,13 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX ./prowler | ansi2html -la > report.html ``` - >Note about output formats to use with `-M`: "text" is the default one with colors, "mono" is like default one but monochrome, "csv" is comma separated values, "json" plain basic json (without comma between lines) and "json-asff" is also json with Amazon Security Finding Format that you can ship to Security Hub using `-S`. + To generate JUnit report files add `-J`. This can be combined with any format. Files are written inside a prowler root directory named `junit-reports`: + + ```sh + ./prowler -J + ``` + + >Note about output formats to use with `-M`: "text" is the default one with colors, "mono" is like default one but monochrome, "csv" is comma separated values, "json" plain basic json (without comma between lines) and "json-asff" is also json with Amazon Security Finding Format that you can ship to Security Hub using `-S`. or save your report in a S3 bucket (this only works for text or mono, for csv, json or json-asff it has to be copied afterwards): @@ -213,7 +232,7 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX 1. For help use: - ``` + ```sh ./prowler -h USAGE: @@ -243,6 +262,7 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX -V show version number & exit -s show scoring report -S send check output to AWS Security Hub - only valid when the output mode is json-asff (i.e. "-M json-asff -S") + -J generate JUnit reports, readable by Jenkins or other CI tools. Files are written to ./junit-reports -x specify external directory with custom checks (i.e. /my/own/checks, files must start by check) -q suppress info messages and passing test output -A account id for the account where to assume a role, requires -R and -T @@ -261,11 +281,11 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX Prowler uses the AWS CLI underneath so it uses the same authentication methods. However, there are few ways to run Prowler against multiple accounts using IAM Assume Role feature depending on eachg use case. You can just set up your custom profile inside `~/.aws/config` with all needed information about the role to assume then call it with `./prowler -p your-custom-profile`. Additionally you can use `-A 123456789012` and `-R RemoteRoleToAssume` and Prowler will get those temporary credentials using `aws sts assume-role`, set them up as environment variables and run against that given account. -``` +```sh ./prowler -A 123456789012 -R ProwlerRole ``` -``` +```sh ./prowler -A 123456789012 -R ProwlerRole -I 123456 ``` @@ -275,11 +295,11 @@ Prowler uses the AWS CLI underneath so it uses the same authentication methods. For example, if you want to get only the fails in CSV format from all checks regarding RDS without banner from the AWS Account 123456789012 assuming the role RemoteRoleToAssume and set a fixed session duration of 1h: -``` +```sh ./prowler -A 123456789012 -R RemoteRoleToAssume -T 3600 -b -M cvs -q -g rds ``` -``` +```sh ./prowler -A 123456789012 -R RemoteRoleToAssume -T 3600 -I 123456 -b -M cvs -q -g rds ``` @@ -304,25 +324,25 @@ Flag `-x /my/own/checks` will include any check in that particular directory. To In order to remove noise and get only FAIL findings there is a `-q` flag that makes Prowler to show and log only FAILs. It can be combined with any other option. -``` +```sh ./prowler -q -M csv -b ``` ## Security Hub integration -Since version v2.3, Prowler supports natively sending findings to [AWS Security Hub](https://aws.amazon.com/security-hub). This integration allows Prowler to import its findings to AWS Security Hub. With Security Hub, you now have a single place that aggregates, organizes, and prioritizes your security alerts, or findings, from multiple AWS services, such as Amazon GuardDuty, Amazon Inspector, Amazon Macie, AWS Identity and Access Management (IAM) Access Analyzer, and AWS Firewall Manager, as well as from AWS Partner solutions and now from Prowler. It is as simple as running the commanbd below: +Since version v2.3, Prowler supports natively sending findings to [AWS Security Hub](https://aws.amazon.com/security-hub). This integration allows Prowler to import its findings to AWS Security Hub. With Security Hub, you now have a single place that aggregates, organizes, and prioritizes your security alerts, or findings, from multiple AWS services, such as Amazon GuardDuty, Amazon Inspector, Amazon Macie, AWS Identity and Access Management (IAM) Access Analyzer, and AWS Firewall Manager, as well as from AWS Partner solutions and now from Prowler. It is as simple as running the command below: + +```sh +./prowler -M json-asff -S +``` - ``` - ./prowler -M json-asff -S - ``` There are two requirements: 1. Security Hub must be enabled for the active region from where you are calling Prowler (if no region is used with `-r` then `us-east-1` is used). It can be enabled by calling `aws securityhub enable-security-hub` 2. As mentioned in section "Custom IAM Policy", to allow Prowler to import its findings to AWS Security Hub you need to add the policy below to the role or user running Prowler: - [iam/prowler-security-hub.json](iam/prowler-security-hub.json) ->Note: to have updated findings in Security Hub you have to run Prowler periodically. Once a day or every certain amount of hours. - +>Note: to have updated findings in Security Hub you have to run Prowler periodically. Once a day or every certain amount of hours. ## How to fix every FAIL @@ -344,7 +364,7 @@ Check your report and fix the issues following all specific guidelines per check If you are using an STS token for AWS-CLI and your session is expired you probably get this error: -``` +```sh A client error (ExpiredToken) occurred when calling the GenerateCredentialReport operation: The security token included in the request is expired ``` @@ -354,16 +374,19 @@ To fix it, please renew your token by authenticating again to the AWS API, see n To run Prowler using a profile that requires MFA you just need to get the session token before hand. Just make sure you use this command: -``` +```sh aws --profile sts get-session-token --duration 129600 --serial-number --token-code --output text - ``` -Once you get your token you can export it as environment variable: ``` + +Once you get your token you can export it as environment variable: + +```sh export AWS_PROFILE=YOUR_AWS_PROFILE export AWS_SESSION_TOKEN=YOUR_NEW_TOKEN AWS_SECRET_ACCESS_KEY=YOUR_SECRET export AWS_ACCESS_KEY_ID=YOUR_KEY ``` + or set manually up your `~/.aws/credentials` file properly. There are some helpfull tools to save time in this process like [aws-mfa-script](https://github.com/asagage/aws-mfa-script) or [aws-cli-mfa](https://github.com/sweharris/aws-cli-mfa). @@ -383,11 +406,13 @@ There are some helpfull tools to save time in this process like [aws-mfa-script] [Prowler-Additions-Policy](iam/prowler-additions-policy.json) Some new and specific checks require Prowler to inherit more permissions than SecurityAudit and ViewOnlyAccess to work properly. In addition to the AWS managed policies, "SecurityAudit" and "ViewOnlyAccess", the user/role you use for checks may need to be granted a custom policy with a few more read-only permissions (to support additional services mostly). Here is an example policy with the additional rights, "Prowler-Additions-Policy" (see below bootstrap script for set it up): + - [iam/prowler-additions-policy.json](iam/prowler-additions-policy.json) [Prowler-Security-Hub Policy](iam/prowler-security-hub.json) Allows Prowler to import its findings to [AWS Security Hub](https://aws.amazon.com/security-hub). More information in [Security Hub integration](#security-hub-integration): + - [iam/prowler-security-hub.json](iam/prowler-security-hub.json) ### Bootstrap Script @@ -418,7 +443,7 @@ Some of these checks look for publicly facing resources may not actually be full To list all existing checks please run the command below: -``` +```sh ./prowler -l ``` @@ -474,6 +499,7 @@ With this group of checks, Prowler shows results of controls related to the "Sec More information on the original PR is [here](https://github.com/toniblyx/prowler/issues/227). ### Note on Business Associate Addendum's (BAA) + Under the HIPAA regulations, cloud service providers (CSPs) such as AWS are considered business associates. The Business Associate Addendum (BAA) is an AWS contract that is required under HIPAA rules to ensure that AWS appropriately safeguards protected health information (PHI). The BAA also serves to clarify and limit, as appropriate, the permissible uses and disclosures of PHI by AWS, based on the relationship between AWS and our customers, and the activities or services being performed by AWS. Customers may use any AWS service in an account designated as a HIPAA account, but they should only process, store, and transmit protected health information (PHI) in the HIPAA-eligible services defined in the Business Associate Addendum (BAA). For the latest list of HIPAA-eligible AWS services, see [HIPAA Eligible Services Reference](https://aws.amazon.com/compliance/hipaa-eligible-services-reference/). More information on AWS & HIPAA can be found [here](https://aws.amazon.com/compliance/hipaa-compliance/) @@ -489,7 +515,9 @@ The `hipaa` group of checks uses existing and extra checks. To get a HIPAA repor ``` ## Trust Boundaries Checks + ### Definition and Terms + The term "trust boundary" is originating from the threat modelling process and the most popular contributor Adam Shostack and author of "Threat Modeling: Designing for Security" defines it as following ([reference](https://adam.shostack.org/uncover.html)): > Trust boundaries are perhaps the most subjective of all: these represent the border between trusted and untrusted elements. Trust is complex. You might trust your mechanic with your car, your dentist with your teeth, and your banker with your money, but you probably don't trust your dentist to change your spark plugs. @@ -498,17 +526,23 @@ AWS is made to be flexible for service links within and between different AWS ac This group of checks helps to analyse a particular AWS account (subject) on existing links to other AWS accounts across various AWS services, in order to identify untrusted links. -### Run +### Run + To give it a quick shot just call: + ```sh ./prowler -g trustboundaries ``` + ### Scenarios + Currently this check group supports two different scenarios: - 1. Single account environment: no action required, the configuration is happening automatically for you. - 2. Multi account environment: in case you environment has multiple trusted and known AWS accounts you maybe want to append them manually to [groups/group16_trustboundaries](groups/group16_trustboundaries) as a space separated list into `GROUP_TRUSTBOUNDARIES_TRUSTED_ACCOUNT_IDS` variable, then just run prowler. + +1. Single account environment: no action required, the configuration is happening automatically for you. +2. Multi account environment: in case you environment has multiple trusted and known AWS accounts you maybe want to append them manually to [groups/group16_trustboundaries](groups/group16_trustboundaries) as a space separated list into `GROUP_TRUSTBOUNDARIES_TRUSTED_ACCOUNT_IDS` variable, then just run prowler. ### Coverage + Current coverage of Amazon Web Service (AWS) taken from [here](https://docs.aws.amazon.com/whitepapers/latest/aws-overview/introduction.html): | Topic | Service | Trust Boundary | |---------------------------------|------------|---------------------------------------------------------------------------| @@ -518,6 +552,7 @@ Current coverage of Amazon Web Service (AWS) taken from [here](https://docs.aws. All ideas or recommendations to extend this group are very welcome [here](https://github.com/toniblyx/prowler/issues/new/choose). ### Detailed Explanation of the Concept + The diagrams depict two common scenarios, single account and multi account environments. Every circle represents one AWS account. The dashed line represents the trust boundary, that separates trust and untrusted AWS accounts. diff --git a/include/junit_integration b/include/junit_integration new file mode 100644 index 00000000..b0134d66 --- /dev/null +++ b/include/junit_integration @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +# Prowler - the handy cloud security tool (copyright 2018) by Toni de la Fuente +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +# Generates JUnit XML reports which can be read by Jenkins or other CI tools + +JUNIT_OUTPUT_DIRECTORY="junit-reports" + +xml_escape() { + sed 's/&/\&/g; s//\>/g; s/\"/\"/g; s/'"'"'/\'/g' <<< "$1" +} + +prepare_junit_output() { + # Remove any JUnit output from previous runs + rm -rf "$JUNIT_OUTPUT_DIRECTORY" + mkdir "$JUNIT_OUTPUT_DIRECTORY" + echo "" + echo "$NOTICE Writing JUnit XML reports to $PROWLER_DIR/$JUNIT_OUTPUT_DIRECTORY $NORMAL" +} + +prepare_junit_check_output() { + # JUnit test cases must be named uniquely, but each Prowler check can output many times due to multiple resources, + # therefore append an index value to the test case name to provide uniqueness, reset it to 1 before starting this check + JUNIT_CHECK_INDEX=1 + # To match JUnit behaviour in Java, and ensure that an aborted execution does not leave a partially written and therefore invalid XML file, + # output a JUnit XML file per check + JUNIT_OUTPUT_FILE="$JUNIT_OUTPUT_DIRECTORY/$1.xml" + printf '%s\n' \ + "" \ + "" \ + " " \ + " " \ + " " \ + " " \ + " " \ + " " \ + " " \ + " " \ + " " \ + " " \ + > "$JUNIT_OUTPUT_FILE" + JUNIT_CHECK_START_TIME=$(get_time_in_milliseconds) +} + +finalise_junit_check_output() { + echo '' >> "$JUNIT_OUTPUT_FILE" +} + +output_junit_success() { + output_junit_test_case "$1" "$(xml_escape "$1")" +} + +output_junit_info() { + # Nothing to output for JUnit for this level of message, but reset the check timer for timing the next check + JUNIT_CHECK_START_TIME=$(get_time_in_milliseconds) +} + +output_junit_failure() { + output_junit_test_case "$1" "" +} + +get_junit_classname() { + #
. naturally follows a Java package structure, so it is suitable as a package name + echo "$TITLE_ID" +} + +output_junit_test_case() { + local time_now + local test_case_duration + time_now=$(get_time_in_milliseconds) + # JUnit test case time values are in seconds, so divide by 1000 using e-3 to convert from milliseconds without losing accuracy due to non-floating point arithmetic + test_case_duration=$(printf "%.3f" "$(("$time_now" - "$JUNIT_CHECK_START_TIME"))e-3") + printf '%s\n' \ + " " \ + " $2" \ + " " >> "$JUNIT_OUTPUT_FILE" + # Reset the check timer for timing the next check + JUNIT_CHECK_START_TIME=$(get_time_in_milliseconds) + ((JUNIT_CHECK_INDEX+=1)) +} diff --git a/include/os_detector b/include/os_detector index 565c6b25..aa071564 100644 --- a/include/os_detector +++ b/include/os_detector @@ -11,17 +11,19 @@ # CONDITIONS OF ANY KIND, either express or implied. See the License for the # specific language governing permissions and limitations under the License. +DATE_CMD="date" + gnu_how_older_from_today() { DATE_TO_COMPARE=$1 - TODAY_IN_DAYS=$(date -d "$(date +%Y-%m-%d)" +%s) - DATE_FROM_IN_DAYS=$(date -d $DATE_TO_COMPARE +%s) + TODAY_IN_DAYS=$("$DATE_CMD" -d "$("$DATE_CMD" +%Y-%m-%d)" +%s) + DATE_FROM_IN_DAYS=$("$DATE_CMD" -d $DATE_TO_COMPARE +%s) DAYS_SINCE=$((($TODAY_IN_DAYS - $DATE_FROM_IN_DAYS )/60/60/24)) echo $DAYS_SINCE } bsd_how_older_from_today() { DATE_TO_COMPARE=$1 - TODAY_IN_DAYS=$(date +%s) - DATE_FROM_IN_DAYS=$(date -jf %Y-%m-%d $DATE_TO_COMPARE +%s) + TODAY_IN_DAYS=$("$DATE_CMD" +%s) + DATE_FROM_IN_DAYS=$("$DATE_CMD" -jf %Y-%m-%d $DATE_TO_COMPARE +%s) DAYS_SINCE=$((($TODAY_IN_DAYS - $DATE_FROM_IN_DAYS )/60/60/24)) echo $DAYS_SINCE } @@ -31,13 +33,13 @@ bsd_how_older_from_today() { gnu_timestamp_to_date() { # remove fractions of a second TIMESTAMP_TO_CONVERT=$(echo $1 | cut -f1 -d".") - OUTPUT_DATE=$(date -d @$TIMESTAMP_TO_CONVERT +'%Y-%m-%d') + OUTPUT_DATE=$("$DATE_CMD" -d @$TIMESTAMP_TO_CONVERT +'%Y-%m-%d') echo $OUTPUT_DATE } bsd_timestamp_to_date() { # remove fractions of a second TIMESTAMP_TO_CONVERT=$(echo $1 | cut -f1 -d".") - OUTPUT_DATE=$(date -r $TIMESTAMP_TO_CONVERT +'%Y-%m-%d') + OUTPUT_DATE=$("$DATE_CMD" -r $TIMESTAMP_TO_CONVERT +'%Y-%m-%d') echo $OUTPUT_DATE } @@ -50,15 +52,15 @@ bsd_decode_report() { gnu_how_many_days_from_today() { DATE_TO_COMPARE=$1 - TODAY_IN_DAYS=$(date -d "$(date +%Y-%m-%d)" +%s) - DATE_IN_DAYS=$(date -d $DATE_TO_COMPARE +%s) + TODAY_IN_DAYS=$("$DATE_CMD" -d "$("$DATE_CMD" +%Y-%m-%d)" +%s) + DATE_IN_DAYS=$("$DATE_CMD" -d $DATE_TO_COMPARE +%s) DAYS_TO=$((( $DATE_IN_DAYS - $TODAY_IN_DAYS )/60/60/24)) echo $DAYS_TO } bsd_how_many_days_from_today() { DATE_TO_COMPARE=$1 - TODAY_IN_DAYS=$(date +%s) - DATE_IN_DAYS=$(date -jf %Y-%m-%d $DATE_TO_COMPARE +%s) + TODAY_IN_DAYS=$("$DATE_CMD" +%s) + DATE_IN_DAYS=$("$DATE_CMD" -jf %Y-%m-%d $DATE_TO_COMPARE +%s) DAYS_TO=$((( $DATE_IN_DAYS - $TODAY_IN_DAYS )/60/60/24)) echo $DAYS_TO } @@ -66,17 +68,32 @@ bsd_how_many_days_from_today() { gnu_get_date_previous_than_months() { MONTHS_TO_COMPARE=$1 MONTHS_TO_COMPARE_IN_SECONDS=$(( 60 * 60 * 24 * 31 * $MONTHS_TO_COMPARE )) - CURRENTSECS=$(date +%s) + CURRENTSECS=$("$DATE_CMD" +%s) STARTDATEINSECS=$(( $CURRENTSECS - $MONTHS_TO_COMPARE_IN_SECONDS )) - DATE_BEFORE_MONTHS_TO_COMPARE=$(date -d @$STARTDATEINSECS '+%Y-%m-%d') + DATE_BEFORE_MONTHS_TO_COMPARE=$("$DATE_CMD" -d @$STARTDATEINSECS '+%Y-%m-%d') echo $DATE_BEFORE_MONTHS_TO_COMPARE } bsd_get_date_previous_than_months() { MONTHS_TO_COMPARE=$1 - DATE_BEFORE_MONTHS_TO_COMPARE=$(date -v -$(echo $MONTHS_TO_COMPARE)m '+%Y-%m-%d') + DATE_BEFORE_MONTHS_TO_COMPARE=$("$DATE_CMD" -v -$(echo $MONTHS_TO_COMPARE)m '+%Y-%m-%d') echo $DATE_BEFORE_MONTHS_TO_COMPARE } +gnu_get_time_in_milliseconds() { + "$DATE_CMD" +%s%3N +} +bsd_get_time_in_milliseconds() { + # BSD date does not support outputting milliseconds, so pad with zeros + "$DATE_CMD" +%s000 +} + +gnu_get_iso8601_timestamp() { + "$DATE_CMD" -u +"%Y-%m-%dT%H:%M:%SZ" +} +bsd_get_iso8601_timestamp() { + "$DATE_CMD" -u +"%Y-%m-%dT%H:%M:%SZ" +} + gnu_test_tcp_connectivity() { HOST=$1 PORT=$2 @@ -114,16 +131,28 @@ if [ "$OSTYPE" == "linux-gnu" ] || [ "$OSTYPE" == "linux-musl" ]; then get_date_previous_than_months() { gnu_get_date_previous_than_months "$1" } + get_time_in_milliseconds() { + gnu_get_time_in_milliseconds + } + get_iso8601_timestamp() { + gnu_get_iso8601_timestamp + } test_tcp_connectivity() { gnu_test_tcp_connectivity "$1" "$2" "$3" } elif [[ "$OSTYPE" == "darwin"* ]]; then # BSD/OSX commands compatibility TEMP_REPORT_FILE=$(mktemp -t prowler.cred_report-XXXXXX) - # It is possible that the user has installed GNU coreutils, replacing the default Mac OS X BSD tools with - # GNU coreutils equivalents. Only GNU date allows --version as a valid argument, so use the validity of this argument + # It is possible that the user has installed GNU coreutils on OS X. By default, this will make GNU commands + # available with a 'g' prefix, e.g. 'gdate'. Test if this is present, and use it if so, as it supports more features. + # The user also may have replaced the default Mac OS X BSD tools with the GNU coreutils equivalents. + # Only GNU date allows --version as a valid argument, so use the validity of this argument # as a means to detect that coreutils is installed and is overriding the default tools - if date --version >/dev/null 2>&1 ; then + GDATE=$(which gdate) + if [ -n "${GDATE}" ]; then + DATE_CMD="gdate" + fi + if "$DATE_CMD" --version >/dev/null 2>&1 ; then how_older_from_today() { gnu_how_older_from_today "$1" } @@ -139,6 +168,12 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then get_date_previous_than_months() { gnu_get_date_previous_than_months "$1" } + get_time_in_milliseconds() { + gnu_get_time_in_milliseconds + } + get_iso8601_timestamp() { + gnu_get_iso8601_timestamp + } else how_older_from_today() { bsd_how_older_from_today "$1" @@ -155,6 +190,12 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then get_date_previous_than_months() { bsd_get_date_previous_than_months "$1" } + get_time_in_milliseconds() { + bsd_get_time_in_milliseconds + } + get_iso8601_timestamp() { + bsd_get_iso8601_timestamp + } fi test_tcp_connectivity() { bsd_test_tcp_connectivity "$1" "$2" "$3" @@ -177,6 +218,12 @@ elif [[ "$OSTYPE" == "cygwin" ]]; then get_date_previous_than_months() { gnu_get_date_previous_than_months "$1" } + get_time_in_milliseconds() { + gnu_get_time_in_milliseconds + } + get_iso8601_timestamp() { + gnu_get_iso8601_timestamp + } test_tcp_connectivity() { gnu_test_tcp_connectivity "$1" "$2" "$3" } diff --git a/include/outputs b/include/outputs index b1e23265..a87f8342 100644 --- a/include/outputs +++ b/include/outputs @@ -16,7 +16,7 @@ EXTENSION_CSV="csv" EXTENSION_JSON="json" EXTENSION_ASFF="asff-json" -EXTENSION_HTML="html" # not implemented yet, use ansi2html as in documentation +EXTENSION_HTML="html" # not implemented yet, use ansi2html as in documentation OUTPUT_DATE=$(date -u +"%Y%m%d%H%M%S") OUTPUT_FILE_NAME=prowler-output-$OUTPUT_DATE @@ -27,6 +27,9 @@ textPass(){ fi PASS_COUNTER=$((PASS_COUNTER+1)) + if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + output_junit_success "$1" + fi if [[ "${MODES[@]}" =~ "csv" || "${MODES[@]}" =~ "json" || "${MODES[@]}" =~ "json-asff" ]]; then if [[ $2 ]]; then REPREGION=$2 @@ -34,14 +37,14 @@ textPass(){ REPREGION=$REGION fi if [[ "${MODES[@]}" =~ "csv" ]]; then - echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}PASS${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV - fi + echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}PASS${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV + fi if [[ "${MODES[@]}" =~ "json" ]]; then - generateJsonOutput "$1" "Pass" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON + generateJsonOutput "$1" "Pass" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON fi if [[ "${MODES[@]}" =~ "json-asff" ]]; then JSON_ASFF_OUTPUT=$(generateJsonAsffOutput "$1" "PASSED" "INFORMATIONAL") - echo "${JSON_ASFF_OUTPUT}" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_ASFF + echo "${JSON_ASFF_OUTPUT}" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_ASFF if [[ "${SEND_TO_SECURITY_HUB}" -eq 1 ]]; then sendToSecurityHub "${JSON_ASFF_OUTPUT}" fi @@ -56,6 +59,9 @@ textInfo(){ return fi + if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + output_junit_info "$1" + fi if [[ "${MODES[@]}" =~ "csv" || "${MODES[@]}" =~ "json" || "${MODES[@]}" =~ "json-asff" ]]; then if [[ $2 ]]; then REPREGION=$2 @@ -76,6 +82,9 @@ textInfo(){ textFail(){ FAIL_COUNTER=$((FAIL_COUNTER+1)) EXITCODE=3 + if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + output_junit_failure "$1" + fi if [[ "${MODES[@]}" =~ "csv" || "${MODES[@]}" =~ "json" || "${MODES[@]}" =~ "json-asff" ]]; then if [[ $2 ]]; then REPREGION=$2 @@ -86,7 +95,7 @@ textFail(){ echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}FAIL${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV fi if [[ "${MODES[@]}" =~ "json" ]]; then - generateJsonOutput "$1" "Fail" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON + generateJsonOutput "$1" "Fail" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON fi if [[ "${MODES[@]}" =~ "json-asff" ]]; then JSON_ASFF_OUTPUT=$(generateJsonAsffOutput "$1" "FAILED" "HIGH") @@ -131,7 +140,7 @@ textTitle(){ esac if [[ "${MODES[@]}" =~ "csv" ]]; then - >&2 echo "$TITLE_ID $TITLE_TEXT" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV + >&2 echo "$TITLE_ID $TITLE_TEXT" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV elif [[ "${MODES[@]}" =~ "json" || "${MODES[@]}" =~ "json-asff" ]]; then : else @@ -156,7 +165,7 @@ generateJsonOutput(){ --arg ITEM_LEVEL "$ITEM_LEVEL" \ --arg TITLE_ID "$TITLE_ID" \ --arg REPREGION "$REPREGION" \ - --arg TIMESTAMP $(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + --arg TIMESTAMP "$(get_iso8601_timestamp)" \ -n '{ "Profile": $PROFILE, "Account Number": $ACCOUNT_NUM, @@ -178,20 +187,17 @@ generateJsonAsffOutput(){ local status=$2 local severity=$3 jq -M -c \ - --arg PROFILE "$PROFILE" \ --arg ACCOUNT_NUM "$ACCOUNT_NUM" \ --arg TITLE_TEXT "$TITLE_TEXT" \ --arg MESSAGE "$(echo -e "${message}" | sed -e 's/^[[:space:]]*//')" \ --arg UNIQUE_ID "$(LC_ALL=C echo -e "${message}" | tr -cs '[:alnum:]._~-\n' '_')" \ --arg STATUS "$status" \ --arg SEVERITY "$severity" \ - --arg SCORED "$ITEM_SCORED" \ - --arg ITEM_LEVEL "$ITEM_LEVEL" \ --arg TITLE_ID "$TITLE_ID" \ --arg TYPE "$ASFF_TYPE" \ --arg RESOURCE_TYPE "$ASFF_RESOURCE_TYPE" \ --arg REPREGION "$REPREGION" \ - --arg TIMESTAMP $(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + --arg TIMESTAMP "$(get_iso8601_timestamp)" \ --arg PROWLER_VERSION "$PROWLER_VERSION" \ -n '{ "SchemaVersion": "2018-10-08", diff --git a/prowler b/prowler index 32b1dbb3..bca29ef0 100755 --- a/prowler +++ b/prowler @@ -45,6 +45,7 @@ SEP=',' KEEPCREDREPORT=0 EXITCODE=0 SEND_TO_SECURITY_HUB=0 +GENERATE_JUNIT=0 SCRIPT_START_TIME=$( date -u +"%Y-%m-%dT%H:%M:%S%z" ) TITLE_ID="" TITLE_TEXT="CALLER ERROR - UNSET TITLE" @@ -78,21 +79,22 @@ USAGE: -V show version number & exit -s show scoring report -S send check output to AWS Security Hub - only valid when the output mode is json-asff (i.e. "-M json-asff -S") + -J generate JUnit reports, readable by Jenkins or other CI tools. Files are written to ./junit-reports -x specify external directory with custom checks (i.e. /my/own/checks, files must start by "check") -q suppress info messages and passing test output -A account id for the account where to assume a role, requires -R and -T (i.e.: 123456789012) -R role name to assume in the account, requires -A and -T (i.e.: ProwlerRole) - -T session durantion given to that role credentials in seconds, default 1h (3600) recommended 12h, requires -R and -T - (i.e.: 43200) - -I External ID to be used when assuming roles (no mandatory), requires -A and -R. + -T session duration given to that role credentials in seconds, default 1h (3600) recommended 12h, requires -R and -T + (i.e.: 43200) + -I External ID to be used when assuming roles (not mandatory), requires -A and -R. -h this help " exit } -while getopts ":hlLkqp:r:c:g:f:m:M:E:enbVsSxI:A:R:T:" OPTION; do +while getopts ":hlLkqp:r:c:g:f:m:M:E:enbVsSJxI:A:R:T:" OPTION; do case $OPTION in h ) usage @@ -152,6 +154,9 @@ while getopts ":hlLkqp:r:c:g:f:m:M:E:enbVsSxI:A:R:T:" OPTION; do S ) SEND_TO_SECURITY_HUB=1 ;; + J ) + GENERATE_JUNIT=1 + ;; x ) EXTERNAL_CHECKS_PATH=$OPTARG ;; @@ -206,6 +211,7 @@ trap "{ rm -f /tmp/prowler*.policy.*; }" EXIT . $PROWLER_DIR/include/assume_role . $PROWLER_DIR/include/connection_tests . $PROWLER_DIR/include/securityhub_integration +. $PROWLER_DIR/include/junit_integration # Get a list of all available AWS Regions REGIONS=$($AWSCLI ec2 describe-regions --query 'Regions[].RegionName' \ @@ -252,8 +258,8 @@ show_group_title() { # Function to execute the check execute_check() { - # See if this is an alternate name for a check - # for example, we might have been passed 1.01 which is another name for 1.1 + # See if this is an alternate name for a check + # for example, we might have been passed 1.01 which is another name for 1.1 local alternate_name_var=CHECK_ALTERNATE_$1 local alternate_name=${!alternate_name_var} # See if this check defines an ASFF Type, if so, use this, falling back to a sane default @@ -266,16 +272,23 @@ execute_check() { ASFF_RESOURCE_TYPE="${!asff_resource_type_var:-AwsAccount}" # Generate the credential report, only if it is group1 related which checks we # run so that the checks can safely assume it's available - if [ ${alternate_name} ];then + if [ ${alternate_name} ];then if [[ ${alternate_name} == check1* || ${alternate_name} == extra71 ]];then if [ ! -s $TEMP_REPORT_FILE ];then genCredReport saveReport fi fi - show_check_title ${alternate_name} - ${alternate_name} - else + show_check_title ${alternate_name} + if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + prepare_junit_check_output "$1" + fi + # Execute the check + ${alternate_name} + if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + finalise_junit_check_output "$1" + fi + else # Check to see if this is a real check local check_id_var=CHECK_ID_$1 local check_id=${!check_id_var} @@ -286,13 +299,20 @@ execute_check() { saveReport fi fi - show_check_title $1 - $1 + show_check_title $1 + if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + prepare_junit_check_output "$1" + fi + # Execute the check + $1 + if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + finalise_junit_check_output "$1" + fi else textFail "ERROR! Use a valid check name (i.e. check41 or extra71)"; exit $EXITCODE fi - fi + fi } # Function to execute all checks in a group @@ -415,7 +435,7 @@ if [[ $PRINTGROUPSONLY == "1" ]]; then fi # Check that jq is installed for JSON outputs -if [[ "$MODE" == "json" || "$MODE" == "json-asff" ]]; then +if [[ ${MODES[@]} =~ "json" || ${MODES[@]} =~ "json-asff" ]]; then . $PROWLER_DIR/include/jq_detector fi @@ -423,6 +443,10 @@ if [[ "$SEND_TO_SECURITY_HUB" -eq 1 ]]; then checkSecurityHubCompatibility fi +if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + prepare_junit_output +fi + # Gather account data / test aws cli connectivity getWhoami From fa17829832019873c587411423df5767b515c046 Mon Sep 17 00:00:00 2001 From: Marc Jay Date: Wed, 15 Apr 2020 12:52:48 +0100 Subject: [PATCH 3/7] Fix arithmetic expression for calculating test duration --- include/junit_integration | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/junit_integration b/include/junit_integration index b0134d66..375531da 100644 --- a/include/junit_integration +++ b/include/junit_integration @@ -78,7 +78,7 @@ output_junit_test_case() { local test_case_duration time_now=$(get_time_in_milliseconds) # JUnit test case time values are in seconds, so divide by 1000 using e-3 to convert from milliseconds without losing accuracy due to non-floating point arithmetic - test_case_duration=$(printf "%.3f" "$(("$time_now" - "$JUNIT_CHECK_START_TIME"))e-3") + test_case_duration=$(printf "%.3f" "$((time_now - JUNIT_CHECK_START_TIME))e-3") printf '%s\n' \ " " \ " $2" \ From dc31adcc18b1b83b41e06f718c49ce67e60aa679 Mon Sep 17 00:00:00 2001 From: Marc Jay Date: Wed, 15 Apr 2020 13:42:33 +0100 Subject: [PATCH 4/7] Rename JUnit XML files to match the Java convention - with a 'TEST-' prefix --- include/junit_integration | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/junit_integration b/include/junit_integration index 375531da..479a7118 100644 --- a/include/junit_integration +++ b/include/junit_integration @@ -33,7 +33,7 @@ prepare_junit_check_output() { JUNIT_CHECK_INDEX=1 # To match JUnit behaviour in Java, and ensure that an aborted execution does not leave a partially written and therefore invalid XML file, # output a JUnit XML file per check - JUNIT_OUTPUT_FILE="$JUNIT_OUTPUT_DIRECTORY/$1.xml" + JUNIT_OUTPUT_FILE="$JUNIT_OUTPUT_DIRECTORY/TEST-$1.xml" printf '%s\n' \ "" \ "" \ From 172f4b2681e3d5aad5cfc416f47c6f7d52ea8c86 Mon Sep 17 00:00:00 2001 From: Alex Gray Date: Wed, 15 Apr 2020 15:19:44 -0400 Subject: [PATCH 5/7] Only check latest version of task definition --- checks/check_extra768 | 7 ++++-- .../get_latest_ecs_task_definition_version.py | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 checks/get_latest_ecs_task_definition_version.py diff --git a/checks/check_extra768 b/checks/check_extra768 index b357c72e..94089008 100644 --- a/checks/check_extra768 +++ b/checks/check_extra768 @@ -23,10 +23,13 @@ extra768(){ # this folder is deleted once this check is finished mkdir $SECRETS_TEMP_FOLDER fi - + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" textInfo "Looking for secrets in ECS task definitions' environment variables across all regions... " for regx in $REGIONS; do - LIST_OF_TASK_DEFINITIONS=$($AWSCLI ecs list-task-definitions $PROFILE_OPT --region $regx --query taskDefinitionArns[*] --output text) + # Get a list of ALL Task Definitions: + $AWSCLI ecs list-task-definitions $PROFILE_OPT --region $regx | jq -r .taskDefinitionArns[] > ALL_TASK_DEFINITIONS.txt + # Filter it down to ONLY the latest version of that task definition: + LIST_OF_TASK_DEFINITIONS=$(python ${DIR}/get_latest_ecs_task_definition_version.py -f ALL_TASK_DEFINITIONS.txt) if [[ $LIST_OF_TASK_DEFINITIONS ]]; then for taskDefinition in $LIST_OF_TASK_DEFINITIONS;do IFS='/' read -r -a splitArn <<< "$taskDefinition" diff --git a/checks/get_latest_ecs_task_definition_version.py b/checks/get_latest_ecs_task_definition_version.py new file mode 100644 index 00000000..d096d6fb --- /dev/null +++ b/checks/get_latest_ecs_task_definition_version.py @@ -0,0 +1,23 @@ +import argparse + +def parseArgs(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('-f', help='file containing list of ecs task definitions', required=True) + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parseArgs() + family = {} + with open(args.f, 'r') as fd: + for line in fd: + l = line.strip() + family_name = l[:l.rfind(':')] + version_int = int(l[l.rfind(':') + 1:]) + if family_name not in family: + family[family_name] = version_int + if family[family_name] < version_int: + family[family_name] = version_int + for family, version in family.items(): + print('{}:{}'.format(family, version)) From 78f649bd6594698d2c8a11e01ed865bd2e21bdd6 Mon Sep 17 00:00:00 2001 From: Marc Jay Date: Wed, 15 Apr 2020 23:36:40 +0100 Subject: [PATCH 6/7] Replace -J flag with junit-xml output format Rearrange output functions so they support outputting text alongside other formats, if specified Add a convenience function for checking if JUnit output is enabled Move monochrome setting into loop so it better supports multiple formats Update README --- README.md | 9 ++-- include/colors | 13 +++-- include/junit_integration | 8 +++ include/outputs | 109 ++++++++++++++++++-------------------- prowler | 19 +++---- 5 files changed, 78 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 4b07afa6..18072315 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX ### Save your reports -1. If you want to save your report for later analysis thare are different ways, natively (supported text, mono, csv, json and json-asff see note below for more info): +1. If you want to save your report for later analysis thare are different ways, natively (supported text, mono, csv, json, json-asff and junit-xml see note below for more info): ```sh ./prowler -M csv @@ -203,10 +203,10 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX ./prowler | ansi2html -la > report.html ``` - To generate JUnit report files add `-J`. This can be combined with any format. Files are written inside a prowler root directory named `junit-reports`: + To generate JUnit report files, include the junit-xml format. This can be combined with any other format. Files are written inside a prowler root directory named `junit-reports`: ```sh - ./prowler -J + ./prowler -M text,junit-xml ``` >Note about output formats to use with `-M`: "text" is the default one with colors, "mono" is like default one but monochrome, "csv" is comma separated values, "json" plain basic json (without comma between lines) and "json-asff" is also json with Amazon Security Finding Format that you can ship to Security Hub using `-S`. @@ -249,7 +249,7 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX -f specify an AWS region to run checks against (i.e.: us-west-1) -m specify the maximum number of items to return for long-running requests (default: 100) - -M output mode: text (default), mono, json, json-asff, csv. They can be used combined comma separated. + -M output mode: text (default), mono, json, json-asff, junit-xml, csv. They can be used combined comma separated. (separator is ","; data is on stdout; progress on stderr). -k keep the credential report -n show check numbers to sort easier @@ -262,7 +262,6 @@ This script has been written in bash using AWS-CLI and it works in Linux and OSX -V show version number & exit -s show scoring report -S send check output to AWS Security Hub - only valid when the output mode is json-asff (i.e. "-M json-asff -S") - -J generate JUnit reports, readable by Jenkins or other CI tools. Files are written to ./junit-reports -x specify external directory with custom checks (i.e. /my/own/checks, files must start by check) -q suppress info messages and passing test output -A account id for the account where to assume a role, requires -R and -T diff --git a/include/colors b/include/colors index 2ae6f77f..7bb9f84e 100644 --- a/include/colors +++ b/include/colors @@ -14,16 +14,15 @@ IFS=',' read -ra MODES <<< "${MODE}" for MODE in "${MODES[@]}"; do - if [[ "$MODE" != "mono" && "$MODE" != "text" && "$MODE" != "csv" && "$MODE" != "json" && "$MODE" != "json-asff" ]]; then - echo -e "${OPTRED}ERROR!$OPTNORMAL Invalid output mode. Choose text, mono, csv, json or json-asff. ./prowler -h for help" + if [[ "$MODE" != "mono" && "$MODE" != "text" && "$MODE" != "csv" && "$MODE" != "json" && "$MODE" != "json-asff" && "$MODE" != "junit-xml" ]]; then + echo -e "${OPTRED}ERROR!$OPTNORMAL Invalid output mode. Choose text, mono, csv, json, json-asff or junit-xml. ./prowler -h for help" EXITCODE=1 exit $EXITCODE fi -done - -if [[ "$MODE" == "mono" || "$MODE" == "csv" || "$MODE" == "json" || "$MODE" == "json-asff" ]]; then - MONOCHROME=1 -fi + if [[ "$MODE" == "mono" || "$MODE" == "csv" || "$MODE" == "json" || "$MODE" == "json-asff" ]]; then + MONOCHROME=1 + fi +done if [[ $MONOCHROME -eq 1 ]]; then # Colors diff --git a/include/junit_integration b/include/junit_integration index 479a7118..54bcd892 100644 --- a/include/junit_integration +++ b/include/junit_integration @@ -15,6 +15,14 @@ JUNIT_OUTPUT_DIRECTORY="junit-reports" +is_junit_output_enabled() { + if [[ ${MODES[@]} =~ "junit-xml" ]]; then + true + else + false + fi +} + xml_escape() { sed 's/&/\&/g; s//\>/g; s/\"/\"/g; s/'"'"'/\'/g' <<< "$1" } diff --git a/include/outputs b/include/outputs index a87f8342..0b56ef88 100644 --- a/include/outputs +++ b/include/outputs @@ -27,29 +27,28 @@ textPass(){ fi PASS_COUNTER=$((PASS_COUNTER+1)) - if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + if [[ $2 ]]; then + REPREGION=$2 + else + REPREGION=$REGION + fi + if [[ "${MODES[@]}" =~ "csv" ]]; then + echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}PASS${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV + fi + if [[ "${MODES[@]}" =~ "json" ]]; then + generateJsonOutput "$1" "Pass" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON + fi + if [[ "${MODES[@]}" =~ "json-asff" ]]; then + JSON_ASFF_OUTPUT=$(generateJsonAsffOutput "$1" "PASSED" "INFORMATIONAL") + echo "${JSON_ASFF_OUTPUT}" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_ASFF + if [[ "${SEND_TO_SECURITY_HUB}" -eq 1 ]]; then + sendToSecurityHub "${JSON_ASFF_OUTPUT}" + fi + fi + if is_junit_output_enabled; then output_junit_success "$1" fi - if [[ "${MODES[@]}" =~ "csv" || "${MODES[@]}" =~ "json" || "${MODES[@]}" =~ "json-asff" ]]; then - if [[ $2 ]]; then - REPREGION=$2 - else - REPREGION=$REGION - fi - if [[ "${MODES[@]}" =~ "csv" ]]; then - echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}PASS${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV - fi - if [[ "${MODES[@]}" =~ "json" ]]; then - generateJsonOutput "$1" "Pass" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON - fi - if [[ "${MODES[@]}" =~ "json-asff" ]]; then - JSON_ASFF_OUTPUT=$(generateJsonAsffOutput "$1" "PASSED" "INFORMATIONAL") - echo "${JSON_ASFF_OUTPUT}" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_ASFF - if [[ "${SEND_TO_SECURITY_HUB}" -eq 1 ]]; then - sendToSecurityHub "${JSON_ASFF_OUTPUT}" - fi - fi - else + if [[ "${MODES[@]}" =~ "text" ]]; then echo " $OK PASS!$NORMAL $1" fi } @@ -59,22 +58,21 @@ textInfo(){ return fi - if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + if [[ $2 ]]; then + REPREGION=$2 + else + REPREGION=$REGION + fi + if [[ "${MODES[@]}" =~ "csv" ]]; then + echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}INFO${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV + fi + if [[ "${MODES[@]}" =~ "json" ]]; then + generateJsonOutput "$1" "Info" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON + fi + if is_junit_output_enabled; then output_junit_info "$1" fi - if [[ "${MODES[@]}" =~ "csv" || "${MODES[@]}" =~ "json" || "${MODES[@]}" =~ "json-asff" ]]; then - if [[ $2 ]]; then - REPREGION=$2 - else - REPREGION=$REGION - fi - if [[ "${MODES[@]}" =~ "csv" ]]; then - echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}INFO${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV - fi - if [[ "${MODES[@]}" =~ "json" ]]; then - generateJsonOutput "$1" "Info" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON - fi - else + if [[ "${MODES[@]}" =~ "text" ]]; then echo " $NOTICE INFO! $1 $NORMAL" fi } @@ -82,29 +80,28 @@ textInfo(){ textFail(){ FAIL_COUNTER=$((FAIL_COUNTER+1)) EXITCODE=3 - if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + if [[ $2 ]]; then + REPREGION=$2 + else + REPREGION=$REGION + fi + if [[ "${MODES[@]}" =~ "csv" ]]; then + echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}FAIL${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV + fi + if [[ "${MODES[@]}" =~ "json" ]]; then + generateJsonOutput "$1" "Fail" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON + fi + if [[ "${MODES[@]}" =~ "json-asff" ]]; then + JSON_ASFF_OUTPUT=$(generateJsonAsffOutput "$1" "FAILED" "HIGH") + echo "${JSON_ASFF_OUTPUT}" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_ASFF + if [[ "${SEND_TO_SECURITY_HUB}" -eq 1 ]]; then + sendToSecurityHub "${JSON_ASFF_OUTPUT}" + fi + fi + if is_junit_output_enabled; then output_junit_failure "$1" fi - if [[ "${MODES[@]}" =~ "csv" || "${MODES[@]}" =~ "json" || "${MODES[@]}" =~ "json-asff" ]]; then - if [[ $2 ]]; then - REPREGION=$2 - else - REPREGION=$REGION - fi - if [[ "${MODES[@]}" =~ "csv" ]]; then - echo "$PROFILE${SEP}$ACCOUNT_NUM${SEP}$REPREGION${SEP}$TITLE_ID${SEP}FAIL${SEP}$ITEM_SCORED${SEP}$ITEM_LEVEL${SEP}$TITLE_TEXT${SEP}$1" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_CSV - fi - if [[ "${MODES[@]}" =~ "json" ]]; then - generateJsonOutput "$1" "Fail" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_JSON - fi - if [[ "${MODES[@]}" =~ "json-asff" ]]; then - JSON_ASFF_OUTPUT=$(generateJsonAsffOutput "$1" "FAILED" "HIGH") - echo "${JSON_ASFF_OUTPUT}" | tee -a $OUTPUT_FILE_NAME.$EXTENSION_ASFF - if [[ "${SEND_TO_SECURITY_HUB}" -eq 1 ]]; then - sendToSecurityHub "${JSON_ASFF_OUTPUT}" - fi - fi - else + if [[ "${MODES[@]}" =~ "text" ]]; then echo " $BAD FAIL! $1 $NORMAL" fi } diff --git a/prowler b/prowler index bca29ef0..f5e097f6 100755 --- a/prowler +++ b/prowler @@ -45,7 +45,6 @@ SEP=',' KEEPCREDREPORT=0 EXITCODE=0 SEND_TO_SECURITY_HUB=0 -GENERATE_JUNIT=0 SCRIPT_START_TIME=$( date -u +"%Y-%m-%dT%H:%M:%S%z" ) TITLE_ID="" TITLE_TEXT="CALLER ERROR - UNSET TITLE" @@ -66,7 +65,7 @@ USAGE: -f specify an AWS region to run checks against (i.e.: us-west-1) -m specify the maximum number of items to return for long-running requests (default: 100) - -M output mode: text (default), mono, json, json-asff, csv. They can be used combined comma separated. + -M output mode: text (default), mono, json, json-asff, junit-xml, csv. They can be used combined comma separated. (separator is ","; data is on stdout; progress on stderr). -k keep the credential report -n show check numbers to sort easier @@ -79,7 +78,6 @@ USAGE: -V show version number & exit -s show scoring report -S send check output to AWS Security Hub - only valid when the output mode is json-asff (i.e. "-M json-asff -S") - -J generate JUnit reports, readable by Jenkins or other CI tools. Files are written to ./junit-reports -x specify external directory with custom checks (i.e. /my/own/checks, files must start by "check") -q suppress info messages and passing test output -A account id for the account where to assume a role, requires -R and -T @@ -94,7 +92,7 @@ USAGE: exit } -while getopts ":hlLkqp:r:c:g:f:m:M:E:enbVsSJxI:A:R:T:" OPTION; do +while getopts ":hlLkqp:r:c:g:f:m:M:E:enbVsSxI:A:R:T:" OPTION; do case $OPTION in h ) usage @@ -154,9 +152,6 @@ while getopts ":hlLkqp:r:c:g:f:m:M:E:enbVsSJxI:A:R:T:" OPTION; do S ) SEND_TO_SECURITY_HUB=1 ;; - J ) - GENERATE_JUNIT=1 - ;; x ) EXTERNAL_CHECKS_PATH=$OPTARG ;; @@ -280,12 +275,12 @@ execute_check() { fi fi show_check_title ${alternate_name} - if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + if is_junit_output_enabled; then prepare_junit_check_output "$1" fi # Execute the check ${alternate_name} - if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + if is_junit_output_enabled; then finalise_junit_check_output "$1" fi else @@ -300,12 +295,12 @@ execute_check() { fi fi show_check_title $1 - if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + if is_junit_output_enabled; then prepare_junit_check_output "$1" fi # Execute the check $1 - if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then + if is_junit_output_enabled; then finalise_junit_check_output "$1" fi else @@ -443,7 +438,7 @@ if [[ "$SEND_TO_SECURITY_HUB" -eq 1 ]]; then checkSecurityHubCompatibility fi -if [[ "${GENERATE_JUNIT}" -eq 1 ]]; then +if is_junit_output_enabled; then prepare_junit_output fi From 5b8370179a136c8a44a905e5f2916e9daf599b69 Mon Sep 17 00:00:00 2001 From: Alex Gray Date: Mon, 20 Apr 2020 09:15:15 -0400 Subject: [PATCH 7/7] Get the list of families and then get latest task definition --- checks/check_extra768 | 22 +++++++++--------- .../get_latest_ecs_task_definition_version.py | 23 ------------------- 2 files changed, 11 insertions(+), 34 deletions(-) delete mode 100644 checks/get_latest_ecs_task_definition_version.py diff --git a/checks/check_extra768 b/checks/check_extra768 index 94089008..591983af 100644 --- a/checks/check_extra768 +++ b/checks/check_extra768 @@ -23,22 +23,22 @@ extra768(){ # this folder is deleted once this check is finished mkdir $SECRETS_TEMP_FOLDER fi - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" textInfo "Looking for secrets in ECS task definitions' environment variables across all regions... " for regx in $REGIONS; do - # Get a list of ALL Task Definitions: - $AWSCLI ecs list-task-definitions $PROFILE_OPT --region $regx | jq -r .taskDefinitionArns[] > ALL_TASK_DEFINITIONS.txt - # Filter it down to ONLY the latest version of that task definition: - LIST_OF_TASK_DEFINITIONS=$(python ${DIR}/get_latest_ecs_task_definition_version.py -f ALL_TASK_DEFINITIONS.txt) - if [[ $LIST_OF_TASK_DEFINITIONS ]]; then - for taskDefinition in $LIST_OF_TASK_DEFINITIONS;do - IFS='/' read -r -a splitArn <<< "$taskDefinition" + # Get a list of all families first: + FAMILIES=$($AWSCLI ecs list-task-definition-families $PROFILE_OPT --region $regx --status ACTIVE | jq -r .families[]) + if [[ $FAMILIES ]]; then + for FAMILY in $FAMILIES;do + # Get the full task definition arn: + TASK_DEFINITION_TEMP=$($AWSCLI ecs list-task-definitions $PROFILE_OPT --region $regx --family-prefix $FAMILY --sort DESC --max-items 1 | jq -r .taskDefinitionArns[0]) + # We only care about the task definition name: + IFS='/' read -r -a splitArn <<< "$TASK_DEFINITION_TEMP" TASK_DEFINITION=${splitArn[1]} TASK_DEFINITION_ENV_VARIABLES_FILE="$SECRETS_TEMP_FOLDER/extra768-$TASK_DEFINITION-$regx-variables.txt" - TASK_DEFINITION_ENV_VARIABLES=$($AWSCLI ecs $PROFILE_OPT --region $regx describe-task-definition --task-definition $taskDefinition --query 'taskDefinition.containerDefinitions[*].environment' --output text > $TASK_DEFINITION_ENV_VARIABLES_FILE) + TASK_DEFINITION_ENV_VARIABLES=$($AWSCLI ecs $PROFILE_OPT --region $regx describe-task-definition --task-definition $TASK_DEFINITION --query 'taskDefinition.containerDefinitions[*].environment' --output text > $TASK_DEFINITION_ENV_VARIABLES_FILE) if [ -s $TASK_DEFINITION_ENV_VARIABLES_FILE ];then - # Implementation using https://github.com/Yelp/detect-secrets - FINDINGS=$(secretsDetector file $TASK_DEFINITION_ENV_VARIABLES_FILE) + # Implementation using https://github.com/Yelp/detect-secrets + FINDINGS=$(secretsDetector file $TASK_DEFINITION_ENV_VARIABLES_FILE) if [[ $FINDINGS -eq 0 ]]; then textPass "$regx: No secrets found in ECS task definition $TASK_DEFINITION variables" "$regx" # delete file if nothing interesting is there diff --git a/checks/get_latest_ecs_task_definition_version.py b/checks/get_latest_ecs_task_definition_version.py deleted file mode 100644 index d096d6fb..00000000 --- a/checks/get_latest_ecs_task_definition_version.py +++ /dev/null @@ -1,23 +0,0 @@ -import argparse - -def parseArgs(): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-f', help='file containing list of ecs task definitions', required=True) - args = parser.parse_args() - return args - - -if __name__ == '__main__': - args = parseArgs() - family = {} - with open(args.f, 'r') as fd: - for line in fd: - l = line.strip() - family_name = l[:l.rfind(':')] - version_int = int(l[l.rfind(':') + 1:]) - if family_name not in family: - family[family_name] = version_int - if family[family_name] < version_int: - family[family_name] = version_int - for family, version in family.items(): - print('{}:{}'.format(family, version))