Skip to content

Feature/803 include other dependencies in vulnerability report #806

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ArBridgeman wants to merge 10 commits into
base: main
Choose a base branch
from
Draft

Feature/803 include other dependencies in vulnerability report #806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
4214475
Refactor out inner function for exporting & update nosec comments
ArBridgeman Apr 24, 2026
59f310d
Move poetry_2_1_pyproject_toml to conftest.py for sharing
ArBridgeman Apr 24, 2026
bb1688b
Expand test to another variant
ArBridgeman Apr 24, 2026
e5d2800
Add sample versions to conftest for later simplified usage
ArBridgeman Apr 24, 2026
2181b07
Simplify setup
ArBridgeman Apr 24, 2026
7c95548
Move fixtures around for simplified usage and prevent local poetry ch…
ArBridgeman Apr 24, 2026
cad38f4
Use new fixture and add optional dependency ruff
ArBridgeman Apr 24, 2026
d4a1a4b
Update poetry dependencies and test
ArBridgeman Apr 24, 2026
b958f5f
Add export plugin to conftest
ArBridgeman Apr 24, 2026
b3e4066
With poetry export fixture
ArBridgeman Apr 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 32 additions & 15 deletions exasol/toolbox/util/dependencies/audit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
r"^Found \d+ known vulnerabilit\w{1,3} in \d+ package\w?$"
)


PipAuditEntry = dict[str, str | list[str] | tuple[str, ...]]


Expand Down Expand Up @@ -159,6 +158,34 @@ def subsection_for_changelog_summary(self) -> str:
""")


def export_dependencies_to_file(output_file: Path, working_directory: Path) -> None:
"""
Export all dependencies to a requirements.txt format

The default for `poetry export` is to only include the main dependencies and their
transitive dependencies, by adding `--all-groups` and `all-extras` we get
all dependencies defined in groups, like dev dependencies, and all optional
dependencies.
"""
command = [
"poetry",
"export",
"--format=requirements.txt",
"--all-groups",
"--all-extras",
]
output = subprocess.run(
command,
capture_output=True,
text=True,
cwd=working_directory,
) # nosec: B603 - allow fixed poetry usage
if output.returncode != 0:
raise PipAuditException.from_subprocess(output, command, cwd=working_directory)

output_file.write_text(output.stdout)


def audit_poetry_files(working_directory: Path) -> str:
"""
Audit the `pyproject.toml` and `poetry.lock` files
Expand All @@ -172,20 +199,10 @@ def audit_poetry_files(working_directory: Path) -> str:
and then inspecting the dependencies.
"""

requirements_txt = "requirements.txt"
command = ["poetry", "export", "--format=requirements.txt"]
output = subprocess.run(
command,
capture_output=True,
text=True,
cwd=working_directory,
) # nosec
if output.returncode != 0:
raise PipAuditException.from_subprocess(output, command, cwd=working_directory)

with tempfile.TemporaryDirectory() as path:
tmpdir = Path(path)
(tmpdir / requirements_txt).write_text(output.stdout)
requirements_path = tmpdir / "requirements.txt"
export_dependencies_to_file(requirements_path, working_directory)

# CLI option `--disable-pip` skips dependency resolution in pip. The
# option can be used with hashed requirements files to avoid
Expand All @@ -195,13 +212,13 @@ def audit_poetry_files(working_directory: Path) -> str:
# In real use scenarios of the PTB we usually have hashed
# requirements. Unfortunately this is not the case for the example
# project created in the integration tests.
command = ["pip-audit", "-r", requirements_txt, "-f", "json"]
command = ["pip-audit", "-r", requirements_path.name, "-f", "json"]
output = subprocess.run(
command,
capture_output=True,
text=True,
cwd=tmpdir,
) # nosec
) # nosec: B603 - allow fixed pip-audit usage

if output.returncode != 0:
# pip-audit does not distinguish between 1) finding vulnerabilities
Expand Down
4 changes: 4 additions & 0 deletions exasol/toolbox/util/dependencies/poetry_dependencies.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,17 @@ def get_section_dict(self, section: str) -> dict | None:
def groups(self) -> tuple[PoetryGroup, ...]:
groups = []

# Main Dependencies
main_key = "project.dependencies"
if self.get_section_dict(main_key):
groups.append(PoetryGroup(name="main", toml_section=main_key))

# Legacy Poetry Main Dependencies
main_dynamic_key = "tool.poetry.dependencies"
if self.get_section_dict(main_dynamic_key):
groups.append(PoetryGroup(name="main", toml_section=main_dynamic_key))

# Legacy Poetry Group Dependencies
group_key = "tool.poetry.group"
if group_dict := self.get_section_dict(group_key):
for group, content in group_dict.items():
Expand All @@ -78,6 +81,7 @@ def groups(self) -> tuple[PoetryGroup, ...]:
)
)

# Poetry Group Dependencies
Copy link
Copy Markdown
Collaborator Author

@ArBridgeman ArBridgeman Apr 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we think it's worthwhile to differentiate main and optional dependency groups, then we will need to make an issue for it.

While we could add snippet to capture optional-dependency groups in this property (PoetryToml.groups). This would be problematic when the groups are then used in PoetryDependencies.direct_dependencies, as the command to extract which groups goes with which dependencies will not work:

[
 "poetry",
 "show",
 "--top-level",
 f"--only={group.name}",
 "--no-truncate
}

In fact, there is not a way to differentiate between main and optional-dependencies from this command, which means current optional-dependencies (when they exist in a project) are already under main. Additionally, as the PEP-compliance of poetry continues to involve, it might be foolhardy for us to try to extract this ourselves from the pyproject.toml, and in the poetry.lock, the markings overlap, meaning one cannot say it is directly from main, transitive, etc.

So it would need to be thought about how one could approach this with minimal effort.

Copy link
Copy Markdown
Collaborator Author

@ArBridgeman ArBridgeman Apr 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See https://github.com/exasol/python-toolbox/pull/806/changes#r3145359536 for a reference of what I described above with optional dependencies being reported in main.

new_group_key = "dependency-groups"
if group_dict := self.get_section_dict(new_group_key):
for group, content in group_dict.items():
Expand Down
23 changes: 23 additions & 0 deletions test/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import json
import os
import subprocess
from inspect import cleandoc

import pytest
Expand All @@ -14,6 +16,27 @@
)


@pytest.fixture(scope="session")
def poetry_path() -> str:
result = subprocess.run(["which", "poetry"], capture_output=True, text=True)
poetry_path = result.stdout.strip()
return poetry_path


@pytest.fixture
def install_poetry_export(poetry_path, monkeypatch):
monkeypatch.setenv("PATH", poetry_path, prepend=os.pathsep)

def _install(cwd):
subprocess.run(
["poetry", "self", "add", "poetry-plugin-export"],
cwd=cwd,
check=True,
)

return _install


class SampleVulnerability:
package_name = "jinja2"
version = "3.1.5"
Expand Down
8 changes: 0 additions & 8 deletions test/integration/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,10 @@
import subprocess
from pathlib import Path

import pytest

from exasol.toolbox.config import BaseConfig


@pytest.fixture(scope="session")
def poetry_path() -> str:
result = subprocess.run(["which", "poetry"], capture_output=True, text=True)
poetry_path = result.stdout.strip()
return poetry_path


@pytest.fixture(scope="session")
def ptb_minimum_python_version() -> str:
"""
Expand Down
59 changes: 58 additions & 1 deletion test/unit/util/dependencies/audit_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import re
from inspect import cleandoc
from pathlib import Path
from subprocess import CompletedProcess
Expand All @@ -13,6 +14,7 @@
Vulnerability,
VulnerabilitySource,
audit_poetry_files,
export_dependencies_to_file,
get_vulnerabilities,
get_vulnerabilities_from_latest_tag,
)
Expand Down Expand Up @@ -94,7 +96,6 @@ def test_reference_links(sample_vulnerability, reference: str, expected: list[st
),
)
def test_vulnerability_id(self, sample_vulnerability, aliases: list[str], expected):

result = Vulnerability(
package=sample_vulnerability.vulnerability.package,
id="DUMMY_IDENTIFIER",
Expand Down Expand Up @@ -124,6 +125,62 @@ def test_subsection_for_changelog_summary(self, sample_vulnerability):
)


@pytest.fixture(scope="module")
def new_pyproject_toml(create_new_poetry_project, project_path):
return (project_path / "pyproject.toml").read_text()


class TestExportDependenciesToFile:
PACKAGES = [
"astroid",
"black", # group - analysis
"click",
"colorama",
"dill",
"isort", # group - dev
"mccabe",
"mypy-extensions",
"packaging",
"pathspec",
"platformdirs",
"pylint", # main
"ruff", # optional-dependencies
"tomli",
"tomlkit",
"typing-extensions",
]

@staticmethod
def extract_package_names(content) -> list[str]:
return re.findall(
r"^([a-zA-Z0-9\-_]+)(?===|>=|<=|>|<|@)", content, re.MULTILINE
)

@pytest.mark.parametrize(
"pyproject_content",
[
"poetry_2_1_pyproject_text",
"poetry_2_3_pyproject_text",
"new_pyproject_toml",
],
)
def test_poetry_export_versions(
self, install_poetry_export, tmp_path, pyproject_content, request
):
content_str = request.getfixturevalue(pyproject_content)
(tmp_path / "pyproject.toml").write_text(content_str)
requirements_txt = tmp_path / "requirements.txt"

install_poetry_export(cwd=tmp_path)

export_dependencies_to_file(
output_file=requirements_txt, working_directory=tmp_path
)

content = requirements_txt.read_text()
assert self.extract_package_names(content) == self.PACKAGES


class TestAuditPoetryFiles:
@staticmethod
@mock.patch("subprocess.run")
Expand Down
114 changes: 114 additions & 0 deletions test/unit/util/dependencies/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import subprocess
from enum import Enum

import pytest


class SampleVersions(str, Enum):
black = "25.1.0"
isort = "6.0.1"
pylint = "3.3.7"
ruff = "0.14.14"


@pytest.fixture(scope="module")
def sample_versions():
return SampleVersions


@pytest.fixture(scope="module")
def poetry_2_1_pyproject_text(sample_versions) -> str:
return f"""
[project]
name = "project"
version = "0.1.0"
description = ""
authors = []
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"pylint (=={sample_versions.pylint})"
]

[tool.poetry]
packages = [{{include = "project", from = "src"}}]

[tool.poetry.group.dev.dependencies]
isort = "{sample_versions.isort}"

[tool.poetry.group.analysis.dependencies]
black = "{sample_versions.black}"

[project.optional-dependencies]
ruff = [ "ruff (=={sample_versions.ruff})" ]

[build-system]
requires = ["poetry-core>=2.0.0,<3.0.0"]
build-backend = "poetry.core.masonry.api"
"""


@pytest.fixture(scope="module")
def poetry_2_3_pyproject_text(sample_versions) -> str:
return f"""
[project]
name = "project"
version = "0.1.0"
description = ""
authors = []
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"pylint (=={sample_versions.pylint})"
]

[tool.poetry]
packages = [{{include = "project", from = "src"}}]

[dependency-groups]
dev = [
"isort=={sample_versions.isort}",
]
analysis = [
"black=={sample_versions.black}"
]

[project.optional-dependencies]
ruff = [ "ruff (=={sample_versions.ruff})" ]

[build-system]
requires = ["poetry-core>=2.0.0,<3.0.0"]
build-backend = "poetry.core.masonry.api"
"""


@pytest.fixture(scope="module")
def cwd(tmp_path_factory):
return tmp_path_factory.mktemp("test")


@pytest.fixture(scope="module")
def project_name():
return "project"


@pytest.fixture(scope="module")
def project_path(cwd, project_name):
return cwd / project_name


@pytest.fixture(scope="module")
def create_new_poetry_project(
poetry_path, cwd, project_name, project_path, sample_versions
):
subprocess.run([poetry_path, "new", project_name], cwd=cwd, check=True)

commands = [
[poetry_path, "self", "add", "poetry-plugin-export"],
[poetry_path, "add", f"pylint=={sample_versions.pylint}"],
[poetry_path, "add", "--group", "dev", f"isort=={sample_versions.isort}"],
[poetry_path, "add", "--group", "analysis", f"black=={sample_versions.black}"],
[poetry_path, "add", f"ruff@{sample_versions.ruff}", "--optional", "ruff"],
]
for cmd in commands:
subprocess.run(cmd, cwd=project_path, env={}, check=True)
Loading
Loading