diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 9ba56f8d6f..867d79c9b6 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" diff --git a/.github/workflows/check-release-tag.yml b/.github/workflows/check-release-tag.yml index 5423c11a4f..f099fa94ee 100644 --- a/.github/workflows/check-release-tag.yml +++ b/.github/workflows/check-release-tag.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a258fef3ac..56dfa2051e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" @@ -42,7 +42,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" @@ -67,7 +67,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: ${{ matrix.python-versions }} poetry-version: "2.3.0" @@ -102,7 +102,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: ${{ matrix.python-versions }} poetry-version: "2.3.0" @@ -128,7 +128,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: ${{ matrix.python-versions }} poetry-version: "2.3.0" @@ -157,7 +157,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" @@ -179,7 +179,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" @@ -188,27 +188,6 @@ jobs: id: build-package run: poetry run -- nox -s package:check - lint-imports: - name: Lint Imports - runs-on: ubuntu-24.04 - permissions: - contents: read - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Lint Imports - id: lint-imports - run: poetry run -- nox -s lint:import - run-unit-tests: name: Unit Tests (Python-${{ matrix.python-versions }}) runs-on: "ubuntu-24.04" @@ -227,7 +206,7 @@ jobs: fetch-depth: 0 - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: ${{ matrix.python-versions }} poetry-version: "2.3.0" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 104045ade7..444c919305 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,14 @@ jobs: permissions: contents: read + dependency-update: + name: Dependency Update + uses: ./.github/workflows/dependency-update.yml + secrets: inherit + permissions: + contents: write + pull-requests: write + report: # Job merge-gate requires manual approval for running the slow checks. If # current workflow ci.yml is triggered by schedule, there is no manual diff --git a/.github/workflows/dependency-update.yml b/.github/workflows/dependency-update.yml new file mode 100644 index 0000000000..f323688ce3 --- /dev/null +++ b/.github/workflows/dependency-update.yml @@ -0,0 +1,108 @@ +name: Dependency Update + +on: + schedule: + # Every Monday at 03:00 UTC + - cron: "0 3 * * 1" + workflow_dispatch: + workflow_call: + +jobs: + dependency-update: + name: Dependency Update + runs-on: "ubuntu-24.04" + permissions: + contents: write + pull-requests: write + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v6 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Audit Dependencies + id: audit-dependencies + run: | + poetry run -- nox -s dependency:audit | tee vulnerabilities.json + LENGTH=$(jq 'length' vulnerabilities.json) + echo "count=$LENGTH" >> "$GITHUB_OUTPUT" + + - name: Update Dependencies + id: update-dependencies + if: steps.audit-dependencies.outputs.count > 0 + run: poetry update + + - name: Check for poetry.lock Changes + id: check-for-poetry-lock-changes + if: steps.audit-dependencies.outputs.count > 0 + run: | + if git diff --quiet -- poetry.lock; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Configure git + id: configure-git + if: steps.check-for-poetry-lock-changes.outputs.changed == 'true' + run: | + git config --global user.email "opensource@exasol.com" + git config --global user.name "Automatic Dependency Updater" + + - name: Create branch + id: create-branch + if: steps.check-for-poetry-lock-changes.outputs.changed == 'true' + run: | + branch_name="dependency-update/$(date "+%Y%m%d%H%M%S")" + echo "Creating branch $branch_name" + git switch -C "$branch_name" + + - name: Commit changes & push + id: publish-branch + if: steps.check-for-poetry-lock-changes.outputs.changed == 'true' + run: | + branch_name=$(git rev-parse --abbrev-ref HEAD) + git add poetry.lock + git commit --message "Update poetry.lock" + git push --set-upstream origin "$branch_name" + + - name: Create pull request + id: create-pr + if: steps.check-for-poetry-lock-changes.outputs.changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + BASE_BRANCH=$(gh repo view --json defaultBranchRef -q .defaultBranchRef.name) + + PR_BODY="Automated dependency update for \`poetry.lock\`. + This PR was created by the dependency update workflow after running: + - \`poetry run -- nox -s dependency:audit\` + - \`poetry update\`" + + PR_URL=$(gh pr create \ + --base "$BASE_BRANCH" \ + --title "Update dependencies to fix vulnerabilities ($(date '+%Y-%m-%d'))" \ + --body "$PR_BODY") + + echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT" + + - name: Report new Pull Request to Slack channel + id: report-pr-slack + if: ${{ steps.create-pr.outputs.pr_url }} + uses: ravsamhq/notify-slack-action@v2 + with: + status: '${{ job.status }}' + token: '${{ secrets.GITHUB_TOKEN }}' + notification_title: 'Dependency update for {repo} created a Pull Request' + message_format: '{workflow} created Pull Request ${{ steps.create-pr.outputs.pr_url }}' + env: + SLACK_WEBHOOK_URL: '${{ secrets.INTEGRATION_TEAM_SECURITY_UPDATES_WEBHOOK }}' diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 147d0453a8..73c74dc99f 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" diff --git a/.github/workflows/matrix-all.yml b/.github/workflows/matrix-all.yml index 69a5aa4b4f..558f5ef672 100644 --- a/.github/workflows/matrix-all.yml +++ b/.github/workflows/matrix-all.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" diff --git a/.github/workflows/matrix-exasol.yml b/.github/workflows/matrix-exasol.yml index 44b5cfd985..a64b88c3f6 100644 --- a/.github/workflows/matrix-exasol.yml +++ b/.github/workflows/matrix-exasol.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" diff --git a/.github/workflows/matrix-python.yml b/.github/workflows/matrix-python.yml index 328799b6c9..e68fd4d1e7 100644 --- a/.github/workflows/matrix-python.yml +++ b/.github/workflows/matrix-python.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 7219a4d83b..46c0129578 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -48,14 +48,6 @@ jobs: permissions: contents: read - test-python-environment: - name: Test python-environment Action - needs: - - approve-run-slow-tests - uses: ./.github/workflows/test-python-environment.yml - permissions: - contents: read - # This job ensures inputs have been executed successfully. allow-merge: name: Allow Merge @@ -66,7 +58,6 @@ jobs: needs: - run-fast-checks - run-slow-checks - - test-python-environment # Each job requires a step, so we added this dummy step. steps: - name: Approve diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 8790ca1ab2..7612aeba7e 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: "3.10" poetry-version: "2.3.0" diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index 8f1b55e42e..46ed19863c 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -29,7 +29,7 @@ jobs: - name: Set up Python & Poetry Environment id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v6 + uses: exasol/python-toolbox/.github/actions/python-environment@v7 with: python-version: ${{ matrix.python-version }} poetry-version: "2.3.0" diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index 366918dc28..87bb81314e 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -59,6 +59,9 @@ Workflows - Pull request and monthly - Executes the continuous integration suite by calling ``merge-gate.yml`` and ``report.yml``. See :ref:`ci_yml` for a graph of workflow calls. + * - ``dependency-update.yml`` + - Weekly and manual + - Audits project dependencies for known vulnerabilities, updates them with Poetry when needed, and creates a pull request if the ``poetry.lock`` was changed. * - ``gh-pages.yml`` - Workflow call - Builds the documentation and deploys it to GitHub Pages. @@ -97,6 +100,17 @@ Workflows CI Actions ---------- +Dependency Update +^^^^^^^^^^^^^^^^^ + +The ``dependency-update.yml`` workflow is used to resolve vulnerabilities by updating our project dependencies. + +It can be triggered manually and is also scheduled to run weekly. + +The workflow first audits dependencies for known vulnerabilities. If vulnerabilities +are detected, it updates the dependencies using Poetry. When the ``poetry.lock`` is changed, +then it creates a pull request with the update. + .. _ci_yml: Pull Request diff --git a/exasol/toolbox/templates/github/workflows/dependency-update.yml b/exasol/toolbox/templates/github/workflows/dependency-update.yml new file mode 100644 index 0000000000..de740a74ed --- /dev/null +++ b/exasol/toolbox/templates/github/workflows/dependency-update.yml @@ -0,0 +1,115 @@ +name: Dependency Update + +on: + schedule: + # Every Monday at 03:00 UTC + - cron: "0 3 * * 1" + workflow_dispatch: + +jobs: + dependency-update: + name: Dependency Update + runs-on: "(( os_version ))" + permissions: + contents: write + pull-requests: write + + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Fail if not running on the default branch + id: check-branch + if: github.ref != format('refs/heads/{0}', github.event.repository.default_branch) + uses: actions/github-script@v8 + with: + script: | + core.setFailed('Not running on the default branch. github.ref is ${{ github.ref }}') + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v6 + with: + python-version: "(( minimum_python_version ))" + poetry-version: "(( dependency_manager_version ))" + + - name: Audit Dependencies + id: audit-dependencies + run: | + poetry run -- nox -s dependency:audit | tee vulnerabilities.json + LENGTH=$(jq 'length' vulnerabilities.json) + echo "count=$LENGTH" >> "$GITHUB_OUTPUT" + + - name: Update Dependencies + id: update-dependencies + if: steps.audit-dependencies.outputs.count > 0 + run: poetry update + + - name: Check for poetry.lock Changes + id: check-for-poetry-lock-changes + if: steps.audit-dependencies.outputs.count > 0 + run: | + if git diff --quiet -- poetry.lock; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Configure git + id: configure-git + if: steps.check-for-poetry-lock-changes.outputs.changed == 'true' + run: | + git config --global user.email "opensource@exasol.com" + git config --global user.name "Automatic Dependency Updater" + + - name: Create branch + id: create-branch + if: steps.check-for-poetry-lock-changes.outputs.changed == 'true' + run: | + branch_name="dependency-update/$(date "+%Y%m%d%H%M%S")" + echo "Creating branch $branch_name" + git switch -C "$branch_name" + + - name: Commit changes & push + id: publish-branch + if: steps.check-for-poetry-lock-changes.outputs.changed == 'true' + run: | + branch_name=$(git rev-parse --abbrev-ref HEAD) + git add poetry.lock + git commit --message "Update poetry.lock" + git push --set-upstream origin "$branch_name" + + - name: Create pull request + id: create-pr + if: steps.check-for-poetry-lock-changes.outputs.changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + BASE_BRANCH=$(gh repo view --json defaultBranchRef -q .defaultBranchRef.name) + + PR_BODY="Automated dependency update for \`poetry.lock\`. + This PR was created by the dependency update workflow after running: + - \`poetry run -- nox -s dependency:audit\` + - \`poetry update\`" + + PR_URL=$(gh pr create \ + --base "$BASE_BRANCH" \ + --title "Update dependencies to fix vulnerabilities ($(date '+%Y-%m-%d'))" \ + --body "$PR_BODY") + + echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT" + + - name: Report new Pull Request to Slack channel + id: report-pr-slack + if: ${{ steps.create-pr.outputs.pr_url }} + uses: ravsamhq/notify-slack-action@v2 + with: + status: '${{ job.status }}' + token: '${{ secrets.GITHUB_TOKEN }}' + notification_title: 'Dependency update for {repo} created a Pull Request' + message_format: '{workflow} created Pull Request ${{ steps.create-pr.outputs.pr_url }}' + env: + SLACK_WEBHOOK_URL: '${{ secrets.INTEGRATION_TEAM_SECURITY_UPDATES_WEBHOOK }}' diff --git a/exasol/toolbox/util/dependencies/audit.py b/exasol/toolbox/util/dependencies/audit.py index 7a18cc3e7c..5d31b3bf98 100644 --- a/exasol/toolbox/util/dependencies/audit.py +++ b/exasol/toolbox/util/dependencies/audit.py @@ -260,7 +260,7 @@ def load_from_pip_audit(cls, working_directory: Path) -> Vulnerabilities: vulnerabilities = [] for entry in audit_dict["dependencies"]: - for vuln_entry in entry["vulns"]: + for vuln_entry in entry.get("vulns", []): vulnerabilities.append( Vulnerability.from_audit_entry( package_name=entry["name"], diff --git a/poetry.lock b/poetry.lock index 18b78aea2e..7ff3669306 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -2698,14 +2698,14 @@ tomli = ">=2.2.1,<3.0.0" [[package]] name = "pytest" -version = "9.0.3" +version = "9.0.0" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9"}, - {file = "pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c"}, + {file = "pytest-9.0.0-py3-none-any.whl", hash = "sha256:e5ccdf10b0bac554970ee88fc1a4ad0ee5d221f8ef22321f9b7e4584e19d7f96"}, + {file = "pytest-9.0.0.tar.gz", hash = "sha256:8f44522eafe4137b0f35c9ce3072931a788a21ee40a2ed279e817d3cc16ed21e"}, ] [package.dependencies] @@ -4038,4 +4038,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "3d5c07aeaab839a92ec06e66addd20d634864518ef66d76623d08d5eaae6817b" +content-hash = "d23e96a6f4ba0858d7603ea5d51e278252545d883c0a7d0577f1036f8cc4220a" diff --git a/pyproject.toml b/pyproject.toml index 516496220d..dbb6fd79f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ "pydantic>=2.11.5,<3", "pylint>=2.15.4", "pysonar>=1.0.1.1548,<2", - "pytest>=7.2.2,<10", + "pytest==9.0.0", "pyupgrade>=2.38.2,<4.0.0", "ruamel-yaml (>=0.18.0,<=0.18.16)", "ruff>=0.14.5,<0.15", diff --git a/test/integration/project-template/nox_test.py b/test/integration/project-template/nox_test.py index 9313f28cba..994e998fa5 100644 --- a/test/integration/project-template/nox_test.py +++ b/test/integration/project-template/nox_test.py @@ -83,4 +83,4 @@ def test_install_github_workflows(self, poetry_path, run_command): assert output.returncode == 0 file_list = run_command(["ls", ".github/workflows"]).stdout.splitlines() - assert len(file_list) == 13 + assert len(file_list) == 14 diff --git a/test/unit/nox/_workflow_test.py b/test/unit/nox/_workflow_test.py index c4a048719c..a0654c93dd 100644 --- a/test/unit/nox/_workflow_test.py +++ b/test/unit/nox/_workflow_test.py @@ -35,7 +35,7 @@ class TestGenerateWorkflow: @staticmethod @pytest.mark.parametrize( "nox_session_runner_posargs, expected_count", - [(ALL, 13), *[(key, 1) for key in WORKFLOW_TEMPLATE_OPTIONS.keys()]], + [(ALL, 14), *[(key, 1) for key in WORKFLOW_TEMPLATE_OPTIONS.keys()]], indirect=["nox_session_runner_posargs"], ) def test_works_as_expected( diff --git a/test/unit/util/dependencies/audit_test.py b/test/unit/util/dependencies/audit_test.py index cc414b0d23..d34bc88f1c 100644 --- a/test/unit/util/dependencies/audit_test.py +++ b/test/unit/util/dependencies/audit_test.py @@ -240,7 +240,13 @@ class TestVulnerabilities: @staticmethod def test_with_no_vulnerabilities(): pip_audit_dict = { - "dependencies": [{"name": "alabaster", "version": "0.7.16", "vulns": []}] + "dependencies": [ + { + "name": "exasol-toolbox", + "skip_reason": "Dependency not found on PyPI and could not be audited: exasol-toolbox (7.0.0)", + }, + {"name": "alabaster", "version": "0.7.16", "vulns": []}, + ] } pip_audit_json = json.dumps(pip_audit_dict) diff --git a/test/unit/util/workflows/templates_test.py b/test/unit/util/workflows/templates_test.py index 994777e261..241796bf39 100644 --- a/test/unit/util/workflows/templates_test.py +++ b/test/unit/util/workflows/templates_test.py @@ -11,6 +11,7 @@ def test_get_workflow_templates(project_config): "check-release-tag", "checks", "ci", + "dependency-update", "gh-pages", "matrix-all", "matrix-exasol",