chore(poetry): make python-poetry as packaging and dependency manager (#1935)

Co-authored-by: Pepe Fagoaga <pepe@verica.io>
2026-02-10 14:55:00 +00:00 · 2023-02-23 11:50:29 +01:00
parent 7a08140a2d
commit 9590e7d7e0
14 changed files with 2689 additions and 1830 deletions
--- a/.github/workflows/build-lint-push-containers.yml
+++ b/.github/workflows/build-lint-push-containers.yml
@@ -47,9 +47,22 @@ jobs:
  container-build:
    # needs: dockerfile-linter
    runs-on: ubuntu-latest
+    env:
+      POETRY_VIRTUALENVS_CREATE: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
+      - name: setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9 #install the python needed
+      - name: Install dependencies
+        run: |
+          pipx install poetry
+          pipx inject poetry poetry-bumpversion
+      - name: Update Prowler version
+        run: |
+          poetry version ${{ github.event.release.tag_name }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -24,9 +24,9 @@ jobs:
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
-          pip install pipenv
-          pipenv install --dev
-          pipenv run pip list
+          pip install poetry
+          poetry install
+          poetry run pip list
          VERSION=$(curl --silent "https://api.github.com/repos/hadolint/hadolint/releases/latest" | \
            grep '"tag_name":' | \
            sed -E 's/.*"v([^"]+)".*/\1/' \
@@ -34,25 +34,25 @@ jobs:
            && chmod +x /tmp/hadolint
      - name: Lint with flake8
        run: |
-          pipenv run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib
+          poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib
      - name: Checking format with black
        run: |
-          pipenv run black --check .
+          poetry run black --check .
      - name: Lint with pylint
        run: |
-          pipenv run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/
+          poetry run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/
      - name: Bandit
        run: |
-          pipenv run bandit -q -lll -x '*_test.py,./contrib/' -r .
+          poetry run bandit -q -lll -x '*_test.py,./contrib/' -r .
      - name: Safety
        run: |
-          pipenv run safety check
+          poetry run safety check
      - name: Vulture
        run: |
-          pipenv run vulture --exclude "contrib" --min-confidence 100 .
+          poetry run vulture --exclude "contrib" --min-confidence 100 .
      - name: Hadolint
        run: |
          /tmp/hadolint Dockerfile --ignore=DL3013
      - name: Test with pytest
        run: |
-          pipenv run pytest tests -n auto
+          poetry run pytest tests -n auto
--- a/.github/workflows/pypi-release.yml
+++ b/.github/workflows/pypi-release.yml
@@ -5,11 +5,14 @@ on:
    types: [published]

 env:
-  GITHUB_BRANCH: ${{ github.event.release.tag_name }}
+  RELEASE_TAG: ${{ github.event.release.tag_name }}
+  GITHUB_BRANCH: master

 jobs:
  release-prowler-job:
    runs-on: ubuntu-latest
+    env:
+      POETRY_VIRTUALENVS_CREATE: "false"
    name: Release Prowler to PyPI
    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -22,20 +25,45 @@ jobs:
          python-version: 3.9 #install the python needed
      - name: Install dependencies
        run: |
-          python -m pip install --upgrade pip
-          pip install build toml --upgrade
-      - name: Build package
-        run: python -m build
-      - name: Publish prowler-cloud package to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
+          pipx install poetry
+          pipx inject poetry poetry-bumpversion
+      - name: Change version and Build package
+        run: |
+          poetry version ${{ env.RELEASE_TAG }}
+          git config user.name "github-actions"
+          git config user.email "<noreply@github.com>"
+          git add prowler/config/config.py pyproject.toml
+          git commit -m "chore(release): ${{ env.RELEASE_TAG }}" --no-verify
+          git tag -fa ${{ env.RELEASE_TAG }} -m "chore(release): ${{ env.RELEASE_TAG }}"
+          git push -f origin ${{ env.RELEASE_TAG }}
+          poetry build
+      - name: Publish prowler package to PyPI
+        run: |
+          poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
+          poetry publish
      - name: Replicate PyPi Package
        run: |
-          rm -rf ./dist && rm -rf ./build && rm -rf prowler_cloud.egg-info
+          rm -rf ./dist && rm -rf ./build && rm -rf prowler.egg-info
          python util/replicate_pypi_package.py
-          python -m build
-      - name: Publish prowler package to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+          poetry build
+      - name: Publish prowler-cloud package to PyPI
+        run: |
+          poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
+          poetry publish
+      # Create pull request with new version
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "chore(release): update Prowler Version to ${{ env.RELEASE_TAG }}."
+          branch: release-${{ env.RELEASE_TAG }}
+          labels: "status/waiting-for-revision, severity/low"
+          title: "chore(release): update Prowler Version to ${{ env.RELEASE_TAG }}"
+          body: |
+            ### Description
+
+            This PR updates Prowler Version to ${{ env.RELEASE_TAG }}.
+
+            ### License
+
+            By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -13,6 +13,13 @@ repos:
      - id: pretty-format-json
        args: ["--autofix", --no-sort-keys, --no-ensure-ascii]

+  ## TOML
+  - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
+    rev: v2.7.0
+    hooks:
+      - id: pretty-format-toml
+        args: [--autofix]
+
  ## BASH
  - repo: https://github.com/koalaman/shellcheck-precommit
    rev: v0.9.0
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -0,0 +1,23 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.9"
+  jobs:
+    post_create_environment:
+      # Install poetry
+      # https://python-poetry.org/docs/#installing-manually
+      - pip install poetry
+      # Tell poetry to not use a virtual environment
+      - poetry config virtualenvs.create false
+    post_install:
+      - poetry install -E docs
+
+mkdocs:
+  configuration: mkdocs.yml
--- a/4
+++ b/4
@@ -24,11 +24,11 @@ lint: ## Lint Code

 ##@ PyPI
 pypi-clean: ## Delete the distribution files
-	rm -rf ./dist && rm -rf ./build && rm -rf prowler_cloud.egg-info
+	rm -rf ./dist && rm -rf ./build && rm -rf prowler.egg-info

 pypi-build: ## Build package
 	$(MAKE) pypi-clean && \
-	python3 -m build
+	poetry build

 pypi-upload: ## Upload package
 	python3 -m twine upload --repository pypi dist/*
--- a/42
+++ b/42
@@ -1,42 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-colorama = "0.4.4"
-boto3 = "1.26.76"
-arnparse = "0.0.2"
-botocore = "1.29.76"
-pydantic = "1.10.5"
-schema = "0.7.5"
-shodan = "1.28.0"
-detect-secrets = "1.4.0"
-alive-progress = "2.4.1"
-tabulate = "0.9.0"
-azure-identity = "1.12.0"
-azure-storage-blob = "12.14.1"
-msgraph-core = "0.2.2"
-azure-mgmt-subscription = "3.1.1"
-azure-mgmt-authorization = "3.0.0"
-azure-mgmt-security = "3.0.0"
-azure-mgmt-storage = "21.0.0"
-
-[dev-packages]
-black = "22.10.0"
-pylint = "2.16.2"
-flake8 = "5.0.4"
-bandit = "1.7.4"
-safety = "2.3.1"
-vulture = "2.7"
-moto = "4.1.3"
-docker = "6.0.0"
-openapi-spec-validator = "0.5.5"
-pytest = "7.2.1"
-pytest-xdist = "3.2.0"
-coverage = "7.1.0"
-sure = "2.0.1"
-freezegun = "1.2.1"
-
-[requires]
-python_version = "3.9"
--- a/Pipfile.lock
+++ b/Pipfile.lock
--- a/README.md
+++ b/README.md
@@ -64,13 +64,13 @@ The container images are available here:

 ## From Github

-Python >= 3.9 is required with pip and pipenv:
+Python >= 3.9 is required with pip and poetry:

 ```
 git clone https://github.com/prowler-cloud/prowler
 cd prowler
-pipenv shell
-pipenv install
+poetry shell
+poetry install
 python prowler.py -v
 ```

--- a/docs/index.md
+++ b/docs/index.md
@@ -5,7 +5,7 @@

 # Prowler Documentation

-**Welcome to [Prowler Open Source v3](https://github.com/prowler-cloud/prowler/) Documentation!** 📄 
+**Welcome to [Prowler Open Source v3](https://github.com/prowler-cloud/prowler/) Documentation!** 📄

 For **Prowler v2 Documentation**, please go [here](https://github.com/prowler-cloud/prowler/tree/2.12.0) to the branch and its README.md.

@@ -118,7 +118,7 @@ Prowler is available as a project in [PyPI](https://pypi.org/project/prowler-clo
    ./configure --enable-optimizations
    sudo make altinstall
    python3.9 --version
-    cd 
+    cd
    ```
    _Commands_:

--- a/poetry.lock
+++ b/poetry.lock
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,66 +1,86 @@
 [build-system]
-requires = ["setuptools>=61.0"]
-build-backend = "setuptools.build_meta"
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core"]

-[project]
-name = "prowler-cloud"
-# https://peps.python.org/pep-0440/
-version = "3.2.1"
-authors = [{ name = "Toni de la Fuente", email = "toni@blyx.com" }]
-maintainers = [
-  { name = "Sergio Garcia", email = "sergio@verica.io" },
-  { name = "Nacho Rivera", email = "nacho@verica.io" },
-  { name = "Pepe Fagoaga", email = "pepe@verica.io" },
-]
-description = "Prowler is an Open Source security tool to perform Cloud Security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains more than 240 controls covering CIS, PCI-DSS, ISO27001, GDPR, HIPAA, FFIEC, SOC2, AWS FTR, ENS and custom security frameworks."
-readme = "README.md"
-requires-python = ">=3.9"
-license = { text = "Apache-2.0" }
+# https://peps.python.org/pep-0621/
+[tool.poetry]
+authors = ["Toni de la Fuente <toni@blyx.com>"]
 classifiers = [
  "Programming Language :: Python :: 3",
  "Programming Language :: Python :: 3.9",
-  "License :: OSI Approved :: Apache Software License",
+  "License :: OSI Approved :: Apache Software License"
 ]
-
-# https://peps.python.org/pep-0631/
-dependencies = [
-  "colorama ~=0.4.4",
-  "boto3 ~=1.26.17",
-  "arnparse ~=0.0.2",
-  "botocore ~=1.29.18",
-  "pydantic ~=1.9.1",
-  "schema ~=0.7.5",
-  "shodan ~=1.28.0",
-  "detect-secrets ~=1.4.0",
-  "alive-progress ~=2.4.1",
-  "tabulate ~=0.9.0",
-  "azure-identity ~=1.12.0",
-  "azure-storage-blob ~=12.14.1",
-  "msgraph-core ~=0.2.2",
-  "azure-mgmt-subscription ~=3.1.1",
-  "azure-mgmt-authorization ~=3.0.0",
-  "azure-mgmt-security ~=3.0.0",
-  "azure-mgmt-storage ~=21.0.0",
+description = "Prowler is an Open Source security tool to perform Cloud Security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains more than 240 controls covering CIS, PCI-DSS, ISO27001, GDPR, HIPAA, FFIEC, SOC2, AWS FTR, ENS and custom security frameworks."
+license = "Apache-2.0"
+maintainers = [
+  "Sergio Garcia <sergio@verica.io>",
+  "Nacho Rivera <nacho@verica.io>",
+  "Pepe Fagoaga <pepe@verica.io>"
 ]
+name = "prowler"
+packages = [
+  {include = "prowler"}
+]
+readme = "README.md"
+version = "3.2.0"

-[project.urls]
-"Homepage" = "https://github.com/prowler-cloud/prowler"
-"Documentation" = "https://docs.prowler.cloud"
-"Issue tracker" = "https://github.com/prowler-cloud/prowler/issues"
-"Changelog" = "https://github.com/prowler-cloud/prowler/releases"
+[tool.poetry.dependencies]
+alive-progress = "2.4.1"
+arnparse = "0.0.2"
+azure-identity = "1.12.0"
+azure-mgmt-authorization = "3.0.0"
+azure-mgmt-security = "3.0.0"
+azure-mgmt-storage = "21.0.0"
+azure-mgmt-subscription = "3.1.1"
+azure-storage-blob = "12.14.1"
+boto3 = "1.26.74"
+botocore = "1.29.74"
+colorama = "0.4.5"
+detect-secrets = "1.4.0"
+mkdocs = {version = "1.4.2", optional = true}
+mkdocs-material = {version = "8.2.1", optional = true}
+msgraph-core = "0.2.2"
+pydantic = "1.10.5"
+python = "^3.9"
+schema = "0.7.5"
+shodan = "1.28.0"
+tabulate = "0.9.0"

-# https://setuptools.pypa.io/en/latest/userguide/package_discovery.html
-[tool.setuptools]
-include-package-data = true
-package-dir = { "prowler" = "prowler" }
+[tool.poetry.extras]
+docs = ["mkdocs", "mkdocs-material"]

-[tool.setuptools.package-data]
-# We need the above pattern to include our data files
-"*" = ["*.yaml", "*.json"]
+[tool.poetry.group.dev.dependencies]
+bandit = "1.7.4"
+black = "22.10.0"
+coverage = "7.1.0"
+docker = "6.0.0"
+flake8 = "5.0.4"
+freezegun = "1.2.1"
+moto = "4.1.2"
+openapi-spec-validator = "0.5.5"
+pylint = "2.16.2"
+pytest = "7.2.1"
+pytest-xdist = "3.2.0"
+safety = "2.3.5"
+sure = "2.0.1"
+vulture = "2.7"

-[project.scripts]
+[tool.poetry.scripts]
 prowler = "prowler.__main__:prowler"

+[tool.poetry.urls]
+"Changelog" = "https://github.com/prowler-cloud/prowler/releases"
+"Documentation" = "https://docs.prowler.cloud"
+"Homepage" = "https://github.com/prowler-cloud/prowler"
+"Issue tracker" = "https://github.com/prowler-cloud/prowler/issues"
+
+[tool.poetry-version-plugin]
+source = "init"
+
+[tool.poetry_bumpversion.file."prowler/config/config.py"]
+replace = 'prowler_version = "{new_version}"'
+search = 'prowler_version = "{current_version}"'
+
 [tool.pytest.ini_options]
 pythonpath = [
  "."
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +0,0 @@
-# Documentation static site generator & deployment tool
-mkdocs>=1.3.0
-mkdocs-material>=8.2.1
--- a/util/replicate_pypi_package.py
+++ b/util/replicate_pypi_package.py
@@ -2,7 +2,7 @@ import toml

 data = toml.load("pyproject.toml")
 # Modify field
-data["project"]["name"] = "prowler"
+data["tool"]["poetry"]["name"] = "prowler-cloud"

 # To use the dump function, you need to open the file in 'write' mode
 f = open("pyproject.toml", "w")