chore(poetry): make python-poetry as packaging and dependency manager (#1935)

Co-authored-by: Pepe Fagoaga <pepe@verica.io>

.github/workflows/build-lint-push-containers.yml

@@ -47,9 +47,22 @@ jobs:
   container-build:
     # needs: dockerfile-linter
     runs-on: ubuntu-latest
+    env:
+      POETRY_VIRTUALENVS_CREATE: "false"
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+      - name: setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9 #install the python needed
+      - name: Install dependencies
+        run: |
+          pipx install poetry
+          pipx inject poetry poetry-bumpversion
+      - name: Update Prowler version
+        run: |
+          poetry version ${{ github.event.release.tag_name }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Build

.github/workflows/pull-request.yml

@@ -24,9 +24,9 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pipenv
+          pip install poetry
-          pipenv install --dev
+          poetry install
-          pipenv run pip list
+          poetry run pip list
           VERSION=$(curl --silent "https://api.github.com/repos/hadolint/hadolint/releases/latest" | \
             grep '"tag_name":' | \
             sed -E 's/.*"v([^"]+)".*/\1/' \
@@ -34,25 +34,25 @@ jobs:
             && chmod +x /tmp/hadolint
       - name: Lint with flake8
         run: |
-          pipenv run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib
+          poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib
       - name: Checking format with black
         run: |
-          pipenv run black --check .
+          poetry run black --check .
       - name: Lint with pylint
         run: |
-          pipenv run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/
+          poetry run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/
       - name: Bandit
         run: |
-          pipenv run bandit -q -lll -x '*_test.py,./contrib/' -r .
+          poetry run bandit -q -lll -x '*_test.py,./contrib/' -r .
       - name: Safety
         run: |
-          pipenv run safety check
+          poetry run safety check
       - name: Vulture
         run: |
-          pipenv run vulture --exclude "contrib" --min-confidence 100 .
+          poetry run vulture --exclude "contrib" --min-confidence 100 .
       - name: Hadolint
         run: |
           /tmp/hadolint Dockerfile --ignore=DL3013
       - name: Test with pytest
         run: |
-          pipenv run pytest tests -n auto
+          poetry run pytest tests -n auto

.github/workflows/pypi-release.yml

@@ -5,11 +5,14 @@ on:
     types: [published]
 
 env:
-  GITHUB_BRANCH: ${{ github.event.release.tag_name }}
+  RELEASE_TAG: ${{ github.event.release.tag_name }}
+  GITHUB_BRANCH: master
 
 jobs:
   release-prowler-job:
     runs-on: ubuntu-latest
+    env:
+      POETRY_VIRTUALENVS_CREATE: "false"
     name: Release Prowler to PyPI
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -22,20 +25,45 @@ jobs:
           python-version: 3.9 #install the python needed
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          pipx install poetry
-          pip install build toml --upgrade
+          pipx inject poetry poetry-bumpversion
-      - name: Build package
+      - name: Change version and Build package
-        run: python -m build
+        run: |
-      - name: Publish prowler-cloud package to PyPI
+          poetry version ${{ env.RELEASE_TAG }}
-        uses: pypa/gh-action-pypi-publish@release/v1
+          git config user.name "github-actions"
-        with:
+          git config user.email "<noreply@github.com>"
-          password: ${{ secrets.PYPI_API_TOKEN }}
+          git add prowler/config/config.py pyproject.toml
+          git commit -m "chore(release): ${{ env.RELEASE_TAG }}" --no-verify
+          git tag -fa ${{ env.RELEASE_TAG }} -m "chore(release): ${{ env.RELEASE_TAG }}"
+          git push -f origin ${{ env.RELEASE_TAG }}
+          poetry build
+      - name: Publish prowler package to PyPI
+        run: |
+          poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
+          poetry publish
       - name: Replicate PyPi Package
         run: |
-          rm -rf ./dist && rm -rf ./build && rm -rf prowler_cloud.egg-info
+          rm -rf ./dist && rm -rf ./build && rm -rf prowler.egg-info
           python util/replicate_pypi_package.py
-          python -m build
+          poetry build
-      - name: Publish prowler package to PyPI
+      - name: Publish prowler-cloud package to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+        run: |
+          poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
+          poetry publish
+      # Create pull request with new version
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
         with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "chore(release): update Prowler Version to ${{ env.RELEASE_TAG }}."
+          branch: release-${{ env.RELEASE_TAG }}
+          labels: "status/waiting-for-revision, severity/low"
+          title: "chore(release): update Prowler Version to ${{ env.RELEASE_TAG }}"
+          body: |
+            ### Description
+
+            This PR updates Prowler Version to ${{ env.RELEASE_TAG }}.
+
+            ### License
+
+            By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

.pre-commit-config.yaml

@@ -13,6 +13,13 @@ repos:
       - id: pretty-format-json
         args: ["--autofix", --no-sort-keys, --no-ensure-ascii]
 
+  ## TOML
+  - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
+    rev: v2.7.0
+    hooks:
+      - id: pretty-format-toml
+        args: [--autofix]
+
   ## BASH
   - repo: https://github.com/koalaman/shellcheck-precommit
     rev: v0.9.0

.readthedocs.yaml

@@ -0,0 +1,23 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.9"
+  jobs:
+    post_create_environment:
+      # Install poetry
+      # https://python-poetry.org/docs/#installing-manually
+      - pip install poetry
+      # Tell poetry to not use a virtual environment
+      - poetry config virtualenvs.create false
+    post_install:
+      - poetry install -E docs
+
+mkdocs:
+  configuration: mkdocs.yml

Makefile

@@ -24,11 +24,11 @@ lint: ## Lint Code
 
 ##@ PyPI
 pypi-clean: ## Delete the distribution files
-	rm -rf ./dist && rm -rf ./build && rm -rf prowler_cloud.egg-info
+	rm -rf ./dist && rm -rf ./build && rm -rf prowler.egg-info
 
 pypi-build: ## Build package
 	$(MAKE) pypi-clean && \
-	python3 -m build
+	poetry build
 
 pypi-upload: ## Upload package
 	python3 -m twine upload --repository pypi dist/*

Pipfile

@@ -1,42 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-colorama = "0.4.4"
-boto3 = "1.26.76"
-arnparse = "0.0.2"
-botocore = "1.29.76"
-pydantic = "1.10.5"
-schema = "0.7.5"
-shodan = "1.28.0"
-detect-secrets = "1.4.0"
-alive-progress = "2.4.1"
-tabulate = "0.9.0"
-azure-identity = "1.12.0"
-azure-storage-blob = "12.14.1"
-msgraph-core = "0.2.2"
-azure-mgmt-subscription = "3.1.1"
-azure-mgmt-authorization = "3.0.0"
-azure-mgmt-security = "3.0.0"
-azure-mgmt-storage = "21.0.0"
-
-[dev-packages]
-black = "22.10.0"
-pylint = "2.16.2"
-flake8 = "5.0.4"
-bandit = "1.7.4"
-safety = "2.3.1"
-vulture = "2.7"
-moto = "4.1.3"
-docker = "6.0.0"
-openapi-spec-validator = "0.5.5"
-pytest = "7.2.1"
-pytest-xdist = "3.2.0"
-coverage = "7.1.0"
-sure = "2.0.1"
-freezegun = "1.2.1"
-
-[requires]
-python_version = "3.9"

README.md

@@ -64,13 +64,13 @@ The container images are available here:
 
 ## From Github
 
-Python >= 3.9 is required with pip and pipenv:
+Python >= 3.9 is required with pip and poetry:
 
 ```
 git clone https://github.com/prowler-cloud/prowler
 cd prowler
-pipenv shell
+poetry shell
-pipenv install
+poetry install
 python prowler.py -v
 ```
 

pyproject.toml

@@ -1,66 +1,86 @@
 [build-system]
-requires = ["setuptools>=61.0"]
+build-backend = "poetry.core.masonry.api"
-build-backend = "setuptools.build_meta"
+requires = ["poetry-core"]
 
-[project]
+# https://peps.python.org/pep-0621/
-name = "prowler-cloud"
+[tool.poetry]
-# https://peps.python.org/pep-0440/
+authors = ["Toni de la Fuente <toni@blyx.com>"]
-version = "3.2.1"
-authors = [{ name = "Toni de la Fuente", email = "toni@blyx.com" }]
-maintainers = [
-  { name = "Sergio Garcia", email = "sergio@verica.io" },
-  { name = "Nacho Rivera", email = "nacho@verica.io" },
-  { name = "Pepe Fagoaga", email = "pepe@verica.io" },
-]
-description = "Prowler is an Open Source security tool to perform Cloud Security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains more than 240 controls covering CIS, PCI-DSS, ISO27001, GDPR, HIPAA, FFIEC, SOC2, AWS FTR, ENS and custom security frameworks."
-readme = "README.md"
-requires-python = ">=3.9"
-license = { text = "Apache-2.0" }
 classifiers = [
   "Programming Language :: Python :: 3",
   "Programming Language :: Python :: 3.9",
-  "License :: OSI Approved :: Apache Software License",
+  "License :: OSI Approved :: Apache Software License"
 ]
-
+description = "Prowler is an Open Source security tool to perform Cloud Security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains more than 240 controls covering CIS, PCI-DSS, ISO27001, GDPR, HIPAA, FFIEC, SOC2, AWS FTR, ENS and custom security frameworks."
-# https://peps.python.org/pep-0631/
+license = "Apache-2.0"
-dependencies = [
+maintainers = [
-  "colorama ~=0.4.4",
+  "Sergio Garcia <sergio@verica.io>",
-  "boto3 ~=1.26.17",
+  "Nacho Rivera <nacho@verica.io>",
-  "arnparse ~=0.0.2",
+  "Pepe Fagoaga <pepe@verica.io>"
-  "botocore ~=1.29.18",
-  "pydantic ~=1.9.1",
-  "schema ~=0.7.5",
-  "shodan ~=1.28.0",
-  "detect-secrets ~=1.4.0",
-  "alive-progress ~=2.4.1",
-  "tabulate ~=0.9.0",
-  "azure-identity ~=1.12.0",
-  "azure-storage-blob ~=12.14.1",
-  "msgraph-core ~=0.2.2",
-  "azure-mgmt-subscription ~=3.1.1",
-  "azure-mgmt-authorization ~=3.0.0",
-  "azure-mgmt-security ~=3.0.0",
-  "azure-mgmt-storage ~=21.0.0",
 ]
+name = "prowler"
+packages = [
+  {include = "prowler"}
+]
+readme = "README.md"
+version = "3.2.0"
 
-[project.urls]
+[tool.poetry.dependencies]
-"Homepage" = "https://github.com/prowler-cloud/prowler"
+alive-progress = "2.4.1"
-"Documentation" = "https://docs.prowler.cloud"
+arnparse = "0.0.2"
-"Issue tracker" = "https://github.com/prowler-cloud/prowler/issues"
+azure-identity = "1.12.0"
-"Changelog" = "https://github.com/prowler-cloud/prowler/releases"
+azure-mgmt-authorization = "3.0.0"
+azure-mgmt-security = "3.0.0"
+azure-mgmt-storage = "21.0.0"
+azure-mgmt-subscription = "3.1.1"
+azure-storage-blob = "12.14.1"
+boto3 = "1.26.74"
+botocore = "1.29.74"
+colorama = "0.4.5"
+detect-secrets = "1.4.0"
+mkdocs = {version = "1.4.2", optional = true}
+mkdocs-material = {version = "8.2.1", optional = true}
+msgraph-core = "0.2.2"
+pydantic = "1.10.5"
+python = "^3.9"
+schema = "0.7.5"
+shodan = "1.28.0"
+tabulate = "0.9.0"
 
-# https://setuptools.pypa.io/en/latest/userguide/package_discovery.html
+[tool.poetry.extras]
-[tool.setuptools]
+docs = ["mkdocs", "mkdocs-material"]
-include-package-data = true
-package-dir = { "prowler" = "prowler" }
 
-[tool.setuptools.package-data]
+[tool.poetry.group.dev.dependencies]
-# We need the above pattern to include our data files
+bandit = "1.7.4"
-"*" = ["*.yaml", "*.json"]
+black = "22.10.0"
+coverage = "7.1.0"
+docker = "6.0.0"
+flake8 = "5.0.4"
+freezegun = "1.2.1"
+moto = "4.1.2"
+openapi-spec-validator = "0.5.5"
+pylint = "2.16.2"
+pytest = "7.2.1"
+pytest-xdist = "3.2.0"
+safety = "2.3.5"
+sure = "2.0.1"
+vulture = "2.7"
 
-[project.scripts]
+[tool.poetry.scripts]
 prowler = "prowler.__main__:prowler"
 
+[tool.poetry.urls]
+"Changelog" = "https://github.com/prowler-cloud/prowler/releases"
+"Documentation" = "https://docs.prowler.cloud"
+"Homepage" = "https://github.com/prowler-cloud/prowler"
+"Issue tracker" = "https://github.com/prowler-cloud/prowler/issues"
+
+[tool.poetry-version-plugin]
+source = "init"
+
+[tool.poetry_bumpversion.file."prowler/config/config.py"]
+replace = 'prowler_version = "{new_version}"'
+search = 'prowler_version = "{current_version}"'
+
 [tool.pytest.ini_options]
 pythonpath = [
   "."

requirements.txt

@@ -1,3 +0,0 @@
-# Documentation static site generator & deployment tool
-mkdocs>=1.3.0
-mkdocs-material>=8.2.1

util/replicate_pypi_package.py

@@ -2,7 +2,7 @@ import toml
 
 data = toml.load("pyproject.toml")
 # Modify field
-data["project"]["name"] = "prowler"
+data["tool"]["poetry"]["name"] = "prowler-cloud"
 
 # To use the dump function, you need to open the file in 'write' mode
 f = open("pyproject.toml", "w")