From d73ece56ebaabc7b2f61c6e1cbc480f16cdb4c98 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 22 Jul 2025 09:12:00 +0200 Subject: [PATCH 01/33] Remove pylint addition which is included already in lint:code --- project-template/{{cookiecutter.repo_name}}/pyproject.toml | 1 - pyproject.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/project-template/{{cookiecutter.repo_name}}/pyproject.toml b/project-template/{{cookiecutter.repo_name}}/pyproject.toml index 1cca979c9..15780e20a 100644 --- a/project-template/{{cookiecutter.repo_name}}/pyproject.toml +++ b/project-template/{{cookiecutter.repo_name}}/pyproject.toml @@ -50,7 +50,6 @@ force_grid_wrap = 2 [tool.pylint.master] fail-under = 5.0 -output-format = "colorized,json:.lint.json,text:.lint.txt" ignore = [] [tool.pylint.format] diff --git a/pyproject.toml b/pyproject.toml index dbbfe848e..e448400c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,6 @@ force_grid_wrap = 2 [tool.pylint.master] fail-under = 7.5 -output-format = "colorized,json:.lint.json,text:.lint.txt" [tool.pylint.format] max-line-length = 88 From 40f0d6ac51cb4e07683c6bdde356d2ce78f248b7 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 22 Jul 2025 09:13:31 +0200 Subject: [PATCH 02/33] Remove unused secrets.GITHUB_TOKEN from checks.yml --- .github/workflows/checks.yml | 2 -- exasol/toolbox/templates/github/workflows/checks.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f2976d5dc..39cfa0204 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -168,8 +168,6 @@ jobs: runs-on: ubuntu-24.04 permissions: contents: read - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} strategy: fail-fast: false matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} diff --git a/exasol/toolbox/templates/github/workflows/checks.yml b/exasol/toolbox/templates/github/workflows/checks.yml index d4ae5cdf1..9e1a6efd4 100644 --- a/exasol/toolbox/templates/github/workflows/checks.yml +++ b/exasol/toolbox/templates/github/workflows/checks.yml @@ -166,8 +166,6 @@ jobs: runs-on: ubuntu-24.04 permissions: contents: read - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} strategy: fail-fast: false matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} From 02e50379a173b14b8e784442e3f1ab99c575829e Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 22 Jul 2025 09:31:22 +0200 Subject: [PATCH 03/33] Add tracking types for DependencyChange with tests Co-authored-by: Christoph Kuhnke --- .../util/dependencies/track_changes.py | 98 +++++++++++++++++++ .../util/dependencies/track_changes_test.py | 93 ++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 exasol/toolbox/util/dependencies/track_changes.py create mode 100644 test/unit/util/dependencies/track_changes_test.py diff --git a/exasol/toolbox/util/dependencies/track_changes.py b/exasol/toolbox/util/dependencies/track_changes.py new file mode 100644 index 000000000..807fe90a6 --- /dev/null +++ b/exasol/toolbox/util/dependencies/track_changes.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from typing import ( + Union, +) + +from packaging.version import Version +from pydantic import ( + BaseModel, + ConfigDict, +) + +from exasol.toolbox.util.dependencies.shared_models import Package + + +class DependencyChange(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + name: str + + +class AddedDependency(DependencyChange): + version: Version + + def __str__(self) -> str: + return f"* Added dependency `{self.name}:{self.version}`" + + @classmethod + def from_package(cls, package: Package) -> AddedDependency: + return cls(name=package.name, version=package.version) + + +class RemovedDependency(DependencyChange): + version: Version + + def __str__(self) -> str: + return f"* Removed dependency `{self.name}:{self.version}`" + + @classmethod + def from_package(cls, package: Package) -> RemovedDependency: + return cls(name=package.name, version=package.version) + + +class UpdatedDependency(DependencyChange): + old_version: Version + current_version: Version + + def __str__(self) -> str: + return ( + f"* Updated dependency `{self.name}:{self.old_version}` " + f"to `{self.current_version}`" + ) + + @classmethod + def from_package( + cls, old_package: Package, current_package: Package + ) -> UpdatedDependency: + return cls( + name=old_package.name, + old_version=old_package.version, + current_version=current_package.version, + ) + + +class DependencyChanges(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + old_dependencies: dict + current_dependencies: dict + + def _categorize_change(self, dependency_name: str) -> Union[DependencyChange, None]: + """ + Categorize dependency change as removed, added, or updated. + """ + old_dependency = self.old_dependencies.get(dependency_name) + current_dependency = self.current_dependencies.get(dependency_name) + if old_dependency and not current_dependency: + return RemovedDependency.from_package(old_dependency) + elif not old_dependency and current_dependency: + return AddedDependency.from_package(current_dependency) + elif old_dependency.version != current_dependency.version: + return UpdatedDependency.from_package(old_dependency, current_dependency) + # dependency was unchanged between versions + return None + + @property + def changes(self) -> list[DependencyChange]: + """ + Return dependency changes + """ + all_dependencies = sorted( + self.old_dependencies.keys() | self.current_dependencies.keys() + ) + return [ + change_dependency + for dependency_name in all_dependencies + if (change_dependency := self._categorize_change(dependency_name)) + ] diff --git a/test/unit/util/dependencies/track_changes_test.py b/test/unit/util/dependencies/track_changes_test.py new file mode 100644 index 000000000..d0b8dc4ff --- /dev/null +++ b/test/unit/util/dependencies/track_changes_test.py @@ -0,0 +1,93 @@ +from exasol.toolbox.util.dependencies.shared_models import Package +from exasol.toolbox.util.dependencies.track_changes import ( + AddedDependency, + DependencyChanges, + RemovedDependency, + UpdatedDependency, +) + + +class SamplePackage: + name = "black" + version = "25.1.0" + + @property + def package(self) -> Package: + return Package(name=self.name, version=self.version) + + @property + def dependency_dict(self) -> dict[str, Package]: + return {self.name: self.package} + + +class TestDependencyChanges: + @staticmethod + def test_removed_dependency(): + changes = DependencyChanges( + old_dependencies=SamplePackage().dependency_dict, current_dependencies={} + ) + + result = changes._categorize_change(SamplePackage.name) + + assert result == RemovedDependency.from_package(SamplePackage().package) + assert str(result) == "* Removed dependency `black:25.1.0`" + + @staticmethod + def test_added_dependency(): + changes = DependencyChanges( + old_dependencies={}, current_dependencies=SamplePackage().dependency_dict + ) + + result = changes._categorize_change(SamplePackage.name) + + assert result == AddedDependency.from_package(SamplePackage().package) + assert str(result) == "* Added dependency `black:25.1.0`" + + @staticmethod + def test_updated_dependency(): + old_package = Package(name=SamplePackage.name, version="24.1.0") + + changes = DependencyChanges( + old_dependencies={SamplePackage.name: old_package}, + current_dependencies=SamplePackage().dependency_dict, + ) + + result = changes._categorize_change(SamplePackage.name) + + assert result == UpdatedDependency.from_package( + old_package=old_package, current_package=SamplePackage().package + ) + assert str(result) == "* Updated dependency `black:24.1.0` to `25.1.0`" + + @staticmethod + def test_dependency_without_changes(): + changes = DependencyChanges( + old_dependencies=SamplePackage().dependency_dict, + current_dependencies=SamplePackage().dependency_dict, + ) + + result = changes._categorize_change(SamplePackage.name) + + assert result is None + + @staticmethod + def test_changes_with_no_changed_dependencies(): + changes = DependencyChanges( + old_dependencies=SamplePackage().dependency_dict, + current_dependencies=SamplePackage().dependency_dict, + ) + + assert changes.changes == [] + + @staticmethod + def test_changes_with_changed_dependencies(): + current_dependencies = SamplePackage().dependency_dict.copy() + added_package = Package(name="pylint", version="3.3.7") + current_dependencies[added_package.name] = added_package + + changes = DependencyChanges( + old_dependencies=SamplePackage().dependency_dict, + current_dependencies=current_dependencies, + ) + + assert changes.changes == [AddedDependency.from_package(added_package)] From 55ac847c3d3b8c0fb494571bcce6740f504bb321 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 24 Jul 2025 10:40:46 +0200 Subject: [PATCH 04/33] Switch dependencies from list to dict of packages for O(1) retrieval --- .../util/dependencies/poetry_dependencies.py | 23 ++++++++++--------- .../dependencies/poetry_dependencies_test.py | 6 ++--- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/exasol/toolbox/util/dependencies/poetry_dependencies.py b/exasol/toolbox/util/dependencies/poetry_dependencies.py index 768f1ffa4..8e4c9ad18 100644 --- a/exasol/toolbox/util/dependencies/poetry_dependencies.py +++ b/exasol/toolbox/util/dependencies/poetry_dependencies.py @@ -92,15 +92,15 @@ def _extract_from_line(line: str) -> Optional[Package]: return None return Package(name=split_line[0], version=split_line[1]) - def _extract_from_poetry_show(self, output_text: str) -> list[Package]: - return [ - package + def _extract_from_poetry_show(self, output_text: str) -> dict[str, Package]: + return { + package.name: package for line in output_text.splitlines() if (package := self._extract_from_line(line)) - ] + } @property - def direct_dependencies(self) -> dict[str, list[Package]]: + def direct_dependencies(self) -> dict[str, dict[str, Package]]: dependencies = {} for group in self.groups: command = ( @@ -122,7 +122,7 @@ def direct_dependencies(self) -> dict[str, list[Package]]: return dependencies @property - def all_dependencies(self) -> dict[str, list[Package]]: + def all_dependencies(self) -> dict[str, dict[str, Package]]: command = ("poetry", "show", "--no-truncate") output = subprocess.run( command, @@ -133,16 +133,17 @@ def all_dependencies(self) -> dict[str, list[Package]]: ) direct_dependencies = self.direct_dependencies.copy() - transitive_dependencies = [] + + transitive_dependencies = {} names_direct_dependencies = { - dep.name - for group_list in direct_dependencies.values() - for dep in group_list + package_name + for group_list in direct_dependencies + for package_name in group_list } for line in output.stdout.splitlines(): dep = self._extract_from_line(line=line) if dep and dep.name not in names_direct_dependencies: - transitive_dependencies.append(dep) + transitive_dependencies[dep.name] = dep return direct_dependencies | {TRANSITIVE_GROUP.name: transitive_dependencies} diff --git a/test/unit/util/dependencies/poetry_dependencies_test.py b/test/unit/util/dependencies/poetry_dependencies_test.py index b44be56cd..f38f8a77a 100644 --- a/test/unit/util/dependencies/poetry_dependencies_test.py +++ b/test/unit/util/dependencies/poetry_dependencies_test.py @@ -23,9 +23,9 @@ BLACK = Package(name="black", version="25.1.0") DIRECT_DEPENDENCIES = { - MAIN_GROUP.name: [PYLINT], - DEV_GROUP.name: [ISORT], - ANALYSIS_GROUP.name: [BLACK], + MAIN_GROUP.name: {PYLINT.name: PYLINT}, + DEV_GROUP.name: {ISORT.name: ISORT}, + ANALYSIS_GROUP.name: {BLACK.name: BLACK}, } From 2a50de18ccc8324f32c3bf01caa1fc75b23a335d Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 24 Jul 2025 12:54:38 +0200 Subject: [PATCH 05/33] Update licenses to work with dictionary and have shared special string type for better readability * Switch to class for cleaner look than function in function --- exasol/toolbox/nox/_dependencies.py | 11 +- exasol/toolbox/util/dependencies/licenses.py | 129 +++++----- .../util/dependencies/poetry_dependencies.py | 25 +- .../util/dependencies/shared_models.py | 15 +- test/unit/util/dependencies/licenses_test.py | 239 +++++++++++------- 5 files changed, 248 insertions(+), 171 deletions(-) diff --git a/exasol/toolbox/nox/_dependencies.py b/exasol/toolbox/nox/_dependencies.py index 57ac71891..adf0167ef 100644 --- a/exasol/toolbox/nox/_dependencies.py +++ b/exasol/toolbox/nox/_dependencies.py @@ -9,8 +9,8 @@ from nox import Session from exasol.toolbox.util.dependencies.licenses import ( - licenses, - packages_to_markdown, + PackageLicenseReport, + get_licenses, ) from exasol.toolbox.util.dependencies.poetry_dependencies import get_dependencies @@ -85,8 +85,11 @@ def run(self, session: Session) -> None: def dependency_licenses(session: Session) -> None: """Return the packages with their licenses""" dependencies = get_dependencies(working_directory=Path()) - package_infos = licenses() - print(packages_to_markdown(dependencies=dependencies, packages=package_infos)) + licenses = get_licenses() + license_markdown = PackageLicenseReport( + dependencies=dependencies, licenses=licenses + ) + print(license_markdown.to_markdown()) @nox.session(name="dependency:audit", python=False) diff --git a/exasol/toolbox/util/dependencies/licenses.py b/exasol/toolbox/util/dependencies/licenses.py index 25a3ac17c..6c1117abb 100644 --- a/exasol/toolbox/util/dependencies/licenses.py +++ b/exasol/toolbox/util/dependencies/licenses.py @@ -2,13 +2,18 @@ import subprocess import tempfile +from dataclasses import dataclass from inspect import cleandoc from json import loads from typing import Optional from pydantic import field_validator -from exasol.toolbox.util.dependencies.shared_models import Package +from exasol.toolbox.util.dependencies.shared_models import ( + NormalizedPackageStr, + Package, + normalize_package_name, +) LICENSE_MAPPING_TO_ABBREVIATION = { "BSD License": "BSD", @@ -88,20 +93,20 @@ def select_most_restrictive(licenses: list[str]) -> str: return LICENSE_MAPPING_TO_ABBREVIATION.get(_license, _license) -def _packages_from_json(json: str) -> list[PackageLicense]: +def _packages_from_json(json: str) -> dict[NormalizedPackageStr, PackageLicense]: packages = loads(json) - return [ - PackageLicense( + return { + normalize_package_name(package["Name"]): PackageLicense( name=package["Name"], package_link=package["URL"], version=package["Version"], license=package["License"], ) for package in packages - ] + } -def licenses() -> list[PackageLicense]: +def get_licenses() -> dict[NormalizedPackageStr, PackageLicense]: with tempfile.NamedTemporaryFile() as file: subprocess.run( [ @@ -117,62 +122,56 @@ def licenses() -> list[PackageLicense]: return _packages_from_json(file.read().decode()) -def packages_to_markdown( - dependencies: dict[str, list], packages: list[PackageLicense] -) -> str: - def heading(): - return "# Dependencies\n" - - def dependency( - group: str, - group_packages: list[Package], - packages: list[PackageLicense], - ) -> str: - def _header(_group: str): - _group = "".join([word.capitalize() for word in _group.strip().split()]) - text = f"## {_group} Dependencies\n" - text += "|Package|Version|License|\n" - text += "|---|---|---|\n" - return text - - def _rows( - _group_packages: list[Package], _packages: list[PackageLicense] - ) -> str: - text = "" - for package in _group_packages: - consistent = filter( - lambda elem: elem.normalized_name == package.normalized_name, - _packages, - ) - for content in consistent: - if content.package_link: - text += f"|[{content.name}]({content.package_link})" - else: - text += f"|{content.name}" - text += f"|{content.version}" - if content.license_link: - text += f"|[{content.license}]({content.license_link})|\n" - else: - text += f"|{content.license}|\n" - text += "\n" - return text - - _template = cleandoc( - """ - {header}{rows} - """ - ) - return _template.format( - header=_header(group), rows=_rows(group_packages, packages) - ) - - template = cleandoc( - """ - {heading}{rows} - """ - ) - - rows = "" - for group in dependencies: - rows += dependency(group, dependencies[group], packages) - return template.format(heading=heading(), rows=rows) +@dataclass(frozen=True) +class PackageLicenseReport: + dependencies: dict[str, dict[NormalizedPackageStr, Package]] + licenses: dict[NormalizedPackageStr, PackageLicense] + + @staticmethod + def _format_group_table_header(group: str): + _group = "".join([word.capitalize() for word in group.strip().split()]) + text = f"## `{group}` Dependencies\n" + text += "|Package|Version|License|\n" + text += "|---|---|---|\n" + return text + + def _format_group_table( + self, group: str, group_package_names: set[NormalizedPackageStr] + ): + group_header = self._format_group_table_header(group=group) + + rows_text = "" + for package_name in group_package_names: + if license_info := self.licenses.get(package_name): + rows_text += self._format_table_row(license_info=license_info) + + return f"""{group_header}{rows_text}\n""" + + @staticmethod + def _format_table_row(license_info: PackageLicense) -> str: + text = "" + # column: package + if license_info.package_link: + text += f"|[{license_info.name}]({license_info.package_link})" + else: + text += f"|{license_info.name}" + + # column: version + text += f"|{license_info.version}" + + # column: license + if license_info.license_link: + text += f"|[{license_info.license}]({license_info.license_link})|" + else: + text += f"|{license_info.license}|" + + return text + "\n" + + def to_markdown(self) -> str: + rows = "" + for group in self.dependencies: + group_package_names = set(self.dependencies[group].keys()) + rows += self._format_group_table( + group=group, group_package_names=group_package_names + ) + return cleandoc(f"""# Dependencies\n\n{rows}""") diff --git a/exasol/toolbox/util/dependencies/poetry_dependencies.py b/exasol/toolbox/util/dependencies/poetry_dependencies.py index 8e4c9ad18..5b1f4f076 100644 --- a/exasol/toolbox/util/dependencies/poetry_dependencies.py +++ b/exasol/toolbox/util/dependencies/poetry_dependencies.py @@ -12,7 +12,10 @@ ) from tomlkit import TOMLDocument -from exasol.toolbox.util.dependencies.shared_models import Package +from exasol.toolbox.util.dependencies.shared_models import ( + NormalizedPackageStr, + Package, +) from exasol.toolbox.util.git import Git @@ -92,15 +95,17 @@ def _extract_from_line(line: str) -> Optional[Package]: return None return Package(name=split_line[0], version=split_line[1]) - def _extract_from_poetry_show(self, output_text: str) -> dict[str, Package]: + def _extract_from_poetry_show( + self, output_text: str + ) -> dict[NormalizedPackageStr, Package]: return { - package.name: package + package.normalized_name: package for line in output_text.splitlines() if (package := self._extract_from_line(line)) } @property - def direct_dependencies(self) -> dict[str, dict[str, Package]]: + def direct_dependencies(self) -> dict[str, dict[NormalizedPackageStr, Package]]: dependencies = {} for group in self.groups: command = ( @@ -122,7 +127,7 @@ def direct_dependencies(self) -> dict[str, dict[str, Package]]: return dependencies @property - def all_dependencies(self) -> dict[str, dict[str, Package]]: + def all_dependencies(self) -> dict[str, dict[NormalizedPackageStr, Package]]: command = ("poetry", "show", "--no-truncate") output = subprocess.run( command, @@ -143,19 +148,23 @@ def all_dependencies(self) -> dict[str, dict[str, Package]]: for line in output.stdout.splitlines(): dep = self._extract_from_line(line=line) if dep and dep.name not in names_direct_dependencies: - transitive_dependencies[dep.name] = dep + transitive_dependencies[dep.normalized_name] = dep return direct_dependencies | {TRANSITIVE_GROUP.name: transitive_dependencies} -def get_dependencies(working_directory: Path) -> dict[str, list[Package]]: +def get_dependencies( + working_directory: Path, +) -> dict[str, dict[NormalizedPackageStr, Package]]: poetry_dep = PoetryToml.load_from_toml(working_directory=working_directory) return PoetryDependencies( groups=poetry_dep.groups, working_directory=working_directory ).direct_dependencies -def get_dependencies_from_latest_tag() -> dict[str, list[Package]]: +def get_dependencies_from_latest_tag() -> ( + dict[str, dict[NormalizedPackageStr, Package]] +): latest_tag = Git.get_latest_tag() with tempfile.TemporaryDirectory() as path: tmpdir = Path(path) diff --git a/exasol/toolbox/util/dependencies/shared_models.py b/exasol/toolbox/util/dependencies/shared_models.py index f7e751458..a4df1e699 100644 --- a/exasol/toolbox/util/dependencies/shared_models.py +++ b/exasol/toolbox/util/dependencies/shared_models.py @@ -1,6 +1,9 @@ from __future__ import annotations -from typing import Annotated +from typing import ( + Annotated, + NewType, +) from packaging.version import Version from pydantic import ( @@ -9,9 +12,15 @@ ConfigDict, ) +NormalizedPackageStr = NewType("NormalizedPackageStr", str) + VERSION_TYPE = Annotated[str, AfterValidator(lambda v: Version(v))] +def normalize_package_name(package_name: str) -> NormalizedPackageStr: + return NormalizedPackageStr(package_name.lower().replace("_", "-")) + + class Package(BaseModel): model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) @@ -19,5 +28,5 @@ class Package(BaseModel): version: VERSION_TYPE @property - def normalized_name(self) -> str: - return self.name.lower().replace("_", "-") + def normalized_name(self) -> NormalizedPackageStr: + return normalize_package_name(self.name) diff --git a/test/unit/util/dependencies/licenses_test.py b/test/unit/util/dependencies/licenses_test.py index bb3e33684..2bdedf65b 100644 --- a/test/unit/util/dependencies/licenses_test.py +++ b/test/unit/util/dependencies/licenses_test.py @@ -1,14 +1,18 @@ +from typing import Union + import pytest from exasol.toolbox.util.dependencies.licenses import ( LICENSE_MAPPING_TO_URL, PackageLicense, + PackageLicenseReport, _normalize, _packages_from_json, - packages_to_markdown, ) from exasol.toolbox.util.dependencies.poetry_dependencies import PoetryGroup -from exasol.toolbox.util.dependencies.shared_models import Package +from exasol.toolbox.util.dependencies.shared_models import ( + Package, +) MAIN_GROUP = PoetryGroup(name="main", toml_section="project.dependencies") DEV_GROUP = PoetryGroup(name="dev", toml_section="tool.poetry.group.dev.dependencies") @@ -62,97 +66,150 @@ def test_normalize(licenses, expected): assert actual == expected -@pytest.mark.parametrize( - "json,expected", - [ - ( - """ - [ - { - "License": "license1", - "Name": "name1", - "URL": "link1", - "Version": "0.1.0" - }, - { - "License": "license2", - "Name": "name2", - "URL": "UNKNOWN", - "Version": "0.2.0" - } - ] - """, - [ - PackageLicense( - name="name1", - version="0.1.0", - package_link="link1", - license="license1", - ), - PackageLicense( - name="name2", - version="0.2.0", - package_link=None, - license="license2", - ), - ], - ) - ], -) -def test_packages_from_json(json, expected): +def test_packages_from_json(): + json = """[ + { + "License": "license1", + "Name": "name1", + "URL": "link1", + "Version": "0.1.0" + }, + { + "License": "license2", + "Name": "name2", + "URL": "UNKNOWN", + "Version": "0.2.0" + } + ] + """ + actual = _packages_from_json(json) - assert actual == expected + + assert actual == { + "name1": PackageLicense( + name="name1", + version="0.1.0", + package_link="link1", + license="license1", + ), + "name2": PackageLicense( + name="name2", + version="0.2.0", + package_link=None, + license="license2", + ), + } -@pytest.mark.parametrize( - "dependencies,packages", - [ - ( - { - MAIN_GROUP.name: [ - Package(name="package1", version="0.1.0"), - Package(name="package3", version="0.1.0"), - ], - DEV_GROUP.name: [Package(name="package2", version="0.2.0")], - }, - [ - PackageLicense( - name="package1", - package_link="package_link1", - version="0.1.0", - license="GPLv1", - ), - PackageLicense( - name="package2", - package_link="package_link2", - version="0.2.0", - license="GPLv2", - ), - PackageLicense( - name="package3", - package_link="UNKNOWN", - version="0.3.0", - license="license3", - ), - ], +@pytest.fixture(scope="class") +def dependencies(): + return { + MAIN_GROUP.name: {"package1": Package(name="package1", version="0.1.0")}, + DEV_GROUP.name: {"package2": Package(name="package2", version="0.2.0")}, + } + + +@pytest.fixture(scope="class") +def package_license_report(dependencies): + licenses = { + "package1": PackageLicense( + name="package1", + package_link="package_link1", + version="0.1.0", + license="GPLv1", + ), + "package2": PackageLicense( + name="package3", + package_link="UNKNOWN", + version="0.3.0", + license="license3", + ), + } + + return PackageLicenseReport(dependencies=dependencies, licenses=licenses) + + +class TestPackageLicenseReport: + @staticmethod + def test_format_group_table_header(package_license_report): + result = package_license_report._format_group_table_header( + group=MAIN_GROUP.name ) - ], -) -def test_packages_to_markdown(dependencies, packages): - actual = packages_to_markdown(dependencies, packages) - assert ( - actual - == """# Dependencies -## Main Dependencies -|Package|Version|License| -|---|---|---| -|[package1](package_link1)|0.1.0|[GPLv1](https://www.gnu.org/licenses/old-licenses/gpl-1.0.html)| -|package3|0.3.0|license3| - -## Dev Dependencies -|Package|Version|License| -|---|---|---| -|[package2](package_link2)|0.2.0|[GPLv2](https://www.gnu.org/licenses/old-licenses/gpl-2.0.html)| - -""" + + assert ( + result + == "## `main` Dependencies\n|Package|Version|License|\n|---|---|---|\n" + ) + + @staticmethod + def test_format_group_table(package_license_report, dependencies): + group_package_names = set(dependencies[MAIN_GROUP.name].keys()) + + result = package_license_report._format_group_table( + group=MAIN_GROUP.name, group_package_names=group_package_names + ) + + assert result == ( + "## `main` Dependencies\n" + "|Package|Version|License|\n" + "|---|---|---|\n" + "|[package1](package_link1)|0.1.0|[GPLv1](https://www.gnu.org/licenses/old-licenses/gpl-1.0.html)|\n" + "\n" + ) + + @staticmethod + @pytest.mark.parametrize( + "package_link,license_link,expected", + [ + pytest.param( + "package_link2", + "GPLv2", + "|[package2](package_link2)|0.2.0|[GPLv2](https://www.gnu.org/licenses/old-licenses/gpl-2.0.html)|\n", + id="has-all-attributes", + ), + pytest.param( + None, + "GPLv2", + "|package2|0.2.0|[GPLv2](https://www.gnu.org/licenses/old-licenses/gpl-2.0.html)|\n", + id="no-package-link", + ), + pytest.param( + "package_link2", + "abcd", + "|[package2](package_link2)|0.2.0|abcd|\n", + id="no-associated-license-link", + ), + ], ) + def test_format_table_row( + package_license_report, + package_link: Union[str, None], + license_link: Union[str, None], + expected, + ): + _license = PackageLicense( + name="package2", + package_link=package_link, + version="0.2.0", + license=license_link, + ) + result = package_license_report._format_table_row(license_info=_license) + + assert result == expected + + @staticmethod + def test_to_markdown(package_license_report): + result = package_license_report.to_markdown() + assert result == ( + "# Dependencies\n" + "\n" + "## `main` Dependencies\n" + "|Package|Version|License|\n" + "|---|---|---|\n" + "|[package1](package_link1)|0.1.0|[GPLv1](https://www.gnu.org/licenses/old-licenses/gpl-1.0.html)|\n" + "\n" + "## `dev` Dependencies\n" + "|Package|Version|License|\n" + "|---|---|---|\n" + "|package3|0.3.0|license3|" + ) From f870236e240db8f27641bbdc261d8d6d097d9413 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 24 Jul 2025 14:59:09 +0200 Subject: [PATCH 06/33] Add dependency updates to changelog --- exasol/toolbox/nox/_release.py | 4 +- .../util/dependencies/poetry_dependencies.py | 9 +- exasol/toolbox/util/release/changelog.py | 56 +++++++++-- test/unit/util/conftest.py | 31 ++++++ test/unit/util/dependencies/licenses_test.py | 27 ++---- test/unit/util/release/changelog_test.py | 94 ++++++++++++++++++- 6 files changed, 181 insertions(+), 40 deletions(-) create mode 100644 test/unit/util/conftest.py diff --git a/exasol/toolbox/nox/_release.py b/exasol/toolbox/nox/_release.py index 42482914e..8236600e9 100644 --- a/exasol/toolbox/nox/_release.py +++ b/exasol/toolbox/nox/_release.py @@ -119,7 +119,9 @@ def prepare_release(session: Session) -> None: _ = _update_project_version(session, new_version) changelogs = Changelogs( - changes_path=PROJECT_CONFIG.doc / "changes", version=new_version + changes_path=PROJECT_CONFIG.doc / "changes", + source_path=PROJECT_CONFIG.root, + version=new_version, ) changelogs.update_changelogs_for_release() changed_files = changelogs.get_changed_files() diff --git a/exasol/toolbox/util/dependencies/poetry_dependencies.py b/exasol/toolbox/util/dependencies/poetry_dependencies.py index 5b1f4f076..97478403c 100644 --- a/exasol/toolbox/util/dependencies/poetry_dependencies.py +++ b/exasol/toolbox/util/dependencies/poetry_dependencies.py @@ -2,6 +2,7 @@ import subprocess import tempfile +from collections import OrderedDict from pathlib import Path from typing import Optional @@ -105,8 +106,10 @@ def _extract_from_poetry_show( } @property - def direct_dependencies(self) -> dict[str, dict[NormalizedPackageStr, Package]]: - dependencies = {} + def direct_dependencies( + self, + ) -> OrderedDict[str, dict[NormalizedPackageStr, Package]]: + dependencies = OrderedDict() for group in self.groups: command = ( "poetry", @@ -127,7 +130,7 @@ def direct_dependencies(self) -> dict[str, dict[NormalizedPackageStr, Package]]: return dependencies @property - def all_dependencies(self) -> dict[str, dict[NormalizedPackageStr, Package]]: + def all_dependencies(self) -> OrderedDict[str, dict[NormalizedPackageStr, Package]]: command = ("poetry", "show", "--no-truncate") output = subprocess.run( command, diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index b63e928e0..d22817f95 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -2,19 +2,26 @@ from datetime import datetime from inspect import cleandoc +from itertools import chain from pathlib import Path +from exasol.toolbox.util.dependencies.poetry_dependencies import ( + get_dependencies, + get_dependencies_from_latest_tag, +) +from exasol.toolbox.util.dependencies.track_changes import DependencyChanges from exasol.toolbox.util.version import Version UNRELEASED_INITIAL_CONTENT = "# Unreleased\n" class Changelogs: - def __init__(self, changes_path: Path, version: Version) -> None: + def __init__(self, changes_path: Path, source_path: Path, version: Version) -> None: self.version = version self.unreleased_md: Path = changes_path / "unreleased.md" self.versioned_changelog_md: Path = changes_path / f"changes_{version}.md" self.changelog_md: Path = changes_path / "changelog.md" + self.source_path: Path = source_path def _create_new_unreleased(self): """ @@ -22,22 +29,21 @@ def _create_new_unreleased(self): """ self.unreleased_md.write_text(UNRELEASED_INITIAL_CONTENT) - def _create_versioned_changelog(self, content: str) -> None: + def _create_versioned_changelog(self, unreleased_content: str) -> None: """ Create a changelog entry for a specific version. Args: - content: The content of the changelog entry. + unreleased_content: The content of the changelog entry. """ - template = cleandoc( - f""" - # {self.version} - {datetime.today().strftime("%Y-%m-%d")} + template = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" + template += f"\n{unreleased_content}" - {content} - """ - ) - self.versioned_changelog_md.write_text(template) + if dependency_content := self._prepare_dependency_update(): + template += f"\n## Dependency Updates\n{dependency_content}" + + self.versioned_changelog_md.write_text(cleandoc(template)) def _extract_unreleased_notes(self) -> str: """ @@ -50,6 +56,36 @@ def _extract_unreleased_notes(self) -> str: unreleased_content += "\n" return unreleased_content + def _prepare_dependency_update(self) -> str: + old_dependencies_in_groups = get_dependencies_from_latest_tag() + current_dependencies_in_groups = get_dependencies( + working_directory=self.source_path + ) + + content = "" + # preserve order of keys from old group + groups = list( + dict.fromkeys( + chain( + old_dependencies_in_groups.keys(), + current_dependencies_in_groups.keys(), + ) + ) + ) + for group in groups: + old_dependencies = old_dependencies_in_groups.get(group, {}) + current_dependencies = current_dependencies_in_groups.get(group, {}) + changes = DependencyChanges( + old_dependencies=old_dependencies, + current_dependencies=current_dependencies, + ).changes + if changes: + # nicer group heading + content += f"\n### `{group}`\n" + content += "\n".join(str(change) for change in changes) + content += "\n" + return content + def _update_changelog_table_of_contents(self) -> None: """ Read in existing `changelog.md` and append to appropriate sections diff --git a/test/unit/util/conftest.py b/test/unit/util/conftest.py new file mode 100644 index 000000000..d1d41a730 --- /dev/null +++ b/test/unit/util/conftest.py @@ -0,0 +1,31 @@ +from collections import OrderedDict + +import pytest + +from exasol.toolbox.util.dependencies.poetry_dependencies import PoetryGroup +from exasol.toolbox.util.dependencies.shared_models import Package + + +@pytest.fixture(scope="module") +def main_group(): + return PoetryGroup(name="main", toml_section="project.dependencies") + + +@pytest.fixture(scope="module") +def dev_group(): + return PoetryGroup(name="dev", toml_section="tool.poetry.group.dev.dependencies") + + +@pytest.fixture(scope="module") +def old_dependencies(main_group, dev_group): + deps = OrderedDict() + deps[main_group.name] = {"package1": Package(name="package1", version="0.0.1")} + return deps + + +@pytest.fixture(scope="module") +def dependencies(main_group, dev_group): + deps = OrderedDict() + deps[main_group.name] = {"package1": Package(name="package1", version="0.1.0")} + deps[dev_group.name] = {"package2": Package(name="package2", version="0.2.0")} + return deps diff --git a/test/unit/util/dependencies/licenses_test.py b/test/unit/util/dependencies/licenses_test.py index 2bdedf65b..f207aaa39 100644 --- a/test/unit/util/dependencies/licenses_test.py +++ b/test/unit/util/dependencies/licenses_test.py @@ -9,13 +9,6 @@ _normalize, _packages_from_json, ) -from exasol.toolbox.util.dependencies.poetry_dependencies import PoetryGroup -from exasol.toolbox.util.dependencies.shared_models import ( - Package, -) - -MAIN_GROUP = PoetryGroup(name="main", toml_section="project.dependencies") -DEV_GROUP = PoetryGroup(name="dev", toml_section="tool.poetry.group.dev.dependencies") class TestPackageLicense: @@ -101,15 +94,7 @@ def test_packages_from_json(): } -@pytest.fixture(scope="class") -def dependencies(): - return { - MAIN_GROUP.name: {"package1": Package(name="package1", version="0.1.0")}, - DEV_GROUP.name: {"package2": Package(name="package2", version="0.2.0")}, - } - - -@pytest.fixture(scope="class") +@pytest.fixture(scope="module") def package_license_report(dependencies): licenses = { "package1": PackageLicense( @@ -131,9 +116,9 @@ def package_license_report(dependencies): class TestPackageLicenseReport: @staticmethod - def test_format_group_table_header(package_license_report): + def test_format_group_table_header(package_license_report, main_group): result = package_license_report._format_group_table_header( - group=MAIN_GROUP.name + group=main_group.name ) assert ( @@ -142,11 +127,11 @@ def test_format_group_table_header(package_license_report): ) @staticmethod - def test_format_group_table(package_license_report, dependencies): - group_package_names = set(dependencies[MAIN_GROUP.name].keys()) + def test_format_group_table(package_license_report, dependencies, main_group): + group_package_names = set(dependencies[main_group.name].keys()) result = package_license_report._format_group_table( - group=MAIN_GROUP.name, group_package_names=group_package_names + group=main_group.name, group_package_names=group_package_names ) assert result == ( diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index 5237d86cf..b1c3ce541 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -1,4 +1,5 @@ from inspect import cleandoc +from unittest import mock import pytest @@ -70,9 +71,35 @@ def unreleased_md(changelogs): ) +@pytest.fixture(scope="function") +def mock_dependencies(dependencies, old_dependencies): + with mock.patch.multiple( + "exasol.toolbox.util.release.changelog", + get_dependencies_from_latest_tag=lambda: old_dependencies, + get_dependencies=lambda working_directory: dependencies, + ): + yield + + +@pytest.fixture(scope="function") +def mock_no_dependencies(): + with mock.patch.multiple( + "exasol.toolbox.util.release.changelog", + get_dependencies_from_latest_tag=lambda: {}, + get_dependencies=lambda working_directory: {}, + ): + yield + + @pytest.fixture(scope="function") def changelogs(tmp_path) -> Changelogs: - return Changelogs(changes_path=tmp_path, version=Version(major=1, minor=0, patch=0)) + changes_path = tmp_path / "doc/changes" + changes_path.mkdir(parents=True) + return Changelogs( + changes_path=changes_path, + source_path=tmp_path, + version=Version(major=1, minor=0, patch=0), + ) class TestChangelogs: @@ -90,7 +117,7 @@ def test_create_new_unreleased(changelogs): assert changelogs.unreleased_md.read_text() == UNRELEASED_INITIAL_CONTENT @staticmethod - def test_create_versioned_changelog(changelogs): + def test_create_versioned_changelog(changelogs, mock_dependencies): changelogs._create_versioned_changelog(SampleContent.changelog) saved_text = changelogs.versioned_changelog_md.read_text() @@ -103,6 +130,18 @@ def test_extract_unreleased_notes(changelogs, unreleased_md): assert result == SampleContent.changelog + "\n" + @staticmethod + def test_prepare_dependency_update(changelogs, mock_dependencies): + result = changelogs._prepare_dependency_update() + assert result == ( + "\n" + "### `main`\n" + "* Updated dependency `package1:0.0.1` to `0.1.0`\n" + "\n" + "### `dev`\n" + "* Added dependency `package2:0.2.0`\n" + ) + @staticmethod def test_update_changelog_table_of_contents(changelogs, changes_md): changelogs._update_changelog_table_of_contents() @@ -110,7 +149,9 @@ def test_update_changelog_table_of_contents(changelogs, changes_md): assert changelogs.changelog_md.read_text() == SampleContent.altered_changes @staticmethod - def test_update_changelogs_for_release(changelogs, unreleased_md, changes_md): + def test_update_changelogs_for_release( + changelogs, mock_dependencies, unreleased_md, changes_md + ): changelogs.update_changelogs_for_release() # changes.md @@ -119,5 +160,48 @@ def test_update_changelogs_for_release(changelogs, unreleased_md, changes_md): assert changelogs.unreleased_md.read_text() == UNRELEASED_INITIAL_CONTENT # versioned.md saved_text = changelogs.versioned_changelog_md.read_text() - assert "1.0.0" in saved_text - assert SampleContent.changelog in saved_text + assert saved_text == cleandoc( + """# 1.0.0 - 2025-07-24 + ## Added + * Added Awesome feature + + ## Changed + * Some behaviour + + ## Fixed + * Fixed nasty bug + + ## Dependency Updates + + ### `main` + * Updated dependency `package1:0.0.1` to `0.1.0` + + ### `dev` + * Added dependency `package2:0.2.0` + """ + ) + + @staticmethod + def test_update_changelogs_for_release_with_no_dependencies( + changelogs, mock_no_dependencies, unreleased_md, changes_md + ): + changelogs.update_changelogs_for_release() + + # changes.md + assert changelogs.changelog_md.read_text() == SampleContent.altered_changes + # unreleased.md + assert changelogs.unreleased_md.read_text() == UNRELEASED_INITIAL_CONTENT + # versioned.md + saved_text = changelogs.versioned_changelog_md.read_text() + assert saved_text == cleandoc( + """# 1.0.0 - 2025-07-24 + ## Added + * Added Awesome feature + + ## Changed + * Some behaviour + + ## Fixed + * Fixed nasty bug + """ + ) From 4077c8823e29aaf49268a8e86c395f8be2c4a3ee Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 24 Jul 2025 15:16:17 +0200 Subject: [PATCH 07/33] Ignore typing as if-elif safeguard None from getting to those lines --- exasol/toolbox/util/dependencies/track_changes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_changes.py b/exasol/toolbox/util/dependencies/track_changes.py index 807fe90a6..875b7cf6e 100644 --- a/exasol/toolbox/util/dependencies/track_changes.py +++ b/exasol/toolbox/util/dependencies/track_changes.py @@ -78,8 +78,8 @@ def _categorize_change(self, dependency_name: str) -> Union[DependencyChange, No return RemovedDependency.from_package(old_dependency) elif not old_dependency and current_dependency: return AddedDependency.from_package(current_dependency) - elif old_dependency.version != current_dependency.version: - return UpdatedDependency.from_package(old_dependency, current_dependency) + elif old_dependency.version != current_dependency.version: # type: ignore + return UpdatedDependency.from_package(old_dependency, current_dependency) # type: ignore # dependency was unchanged between versions return None From 0517262631d3d7e71ac631f6fab980bedafdf277 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 24 Jul 2025 15:18:39 +0200 Subject: [PATCH 08/33] Add changelog entry --- doc/changes/unreleased.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 1c76e6060..92b44b712 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1,5 +1,9 @@ # Unreleased +## Feature + +* #382: Added onto nox session `release:prepare` to append dependency changes between current and latest tag + ## Refactoring * #498: Centralized changelog code relevant for `release:trigger` & robustly tested From 3a8243d5a970277005b7361049757ed9af0451f9 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 24 Jul 2025 15:33:21 +0200 Subject: [PATCH 09/33] Pass on OrderedDict type --- exasol/toolbox/util/dependencies/poetry_dependencies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol/toolbox/util/dependencies/poetry_dependencies.py b/exasol/toolbox/util/dependencies/poetry_dependencies.py index 97478403c..00faadaab 100644 --- a/exasol/toolbox/util/dependencies/poetry_dependencies.py +++ b/exasol/toolbox/util/dependencies/poetry_dependencies.py @@ -158,7 +158,7 @@ def all_dependencies(self) -> OrderedDict[str, dict[NormalizedPackageStr, Packag def get_dependencies( working_directory: Path, -) -> dict[str, dict[NormalizedPackageStr, Package]]: +) -> OrderedDict[str, dict[NormalizedPackageStr, Package]]: poetry_dep = PoetryToml.load_from_toml(working_directory=working_directory) return PoetryDependencies( groups=poetry_dep.groups, working_directory=working_directory @@ -166,7 +166,7 @@ def get_dependencies( def get_dependencies_from_latest_tag() -> ( - dict[str, dict[NormalizedPackageStr, Package]] + OrderedDict[str, dict[NormalizedPackageStr, Package]] ): latest_tag = Git.get_latest_tag() with tempfile.TemporaryDirectory() as path: From 4cb1bf64a40b22347a64bc759d79413f33324344 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 24 Jul 2025 15:34:36 +0200 Subject: [PATCH 10/33] Pass on OrderedDict type --- exasol/toolbox/util/dependencies/licenses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exasol/toolbox/util/dependencies/licenses.py b/exasol/toolbox/util/dependencies/licenses.py index 6c1117abb..2a8d411e4 100644 --- a/exasol/toolbox/util/dependencies/licenses.py +++ b/exasol/toolbox/util/dependencies/licenses.py @@ -2,6 +2,7 @@ import subprocess import tempfile +from collections import OrderedDict from dataclasses import dataclass from inspect import cleandoc from json import loads @@ -124,7 +125,7 @@ def get_licenses() -> dict[NormalizedPackageStr, PackageLicense]: @dataclass(frozen=True) class PackageLicenseReport: - dependencies: dict[str, dict[NormalizedPackageStr, Package]] + dependencies: OrderedDict[str, dict[NormalizedPackageStr, Package]] licenses: dict[NormalizedPackageStr, PackageLicense] @staticmethod From c84bde9bec762dc6f579df142b658fd452b64060 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 24 Jul 2025 15:46:40 +0200 Subject: [PATCH 11/33] Restore order to licenses as dict keys to set loses order --- exasol/toolbox/util/dependencies/licenses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/toolbox/util/dependencies/licenses.py b/exasol/toolbox/util/dependencies/licenses.py index 2a8d411e4..059f8e7d8 100644 --- a/exasol/toolbox/util/dependencies/licenses.py +++ b/exasol/toolbox/util/dependencies/licenses.py @@ -142,7 +142,7 @@ def _format_group_table( group_header = self._format_group_table_header(group=group) rows_text = "" - for package_name in group_package_names: + for package_name in sorted(group_package_names): if license_info := self.licenses.get(package_name): rows_text += self._format_table_row(license_info=license_info) From 983afcab3e81859e0e5ef459b1d7b1d2027d56ea Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 09:49:24 +0200 Subject: [PATCH 12/33] Fix indent and make date variable; could alternately add freeze gun or mock datetime --- test/unit/util/release/changelog_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index b1c3ce541..fac7064e5 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -1,3 +1,4 @@ +from datetime import datetime from inspect import cleandoc from unittest import mock @@ -161,7 +162,7 @@ def test_update_changelogs_for_release( # versioned.md saved_text = changelogs.versioned_changelog_md.read_text() assert saved_text == cleandoc( - """# 1.0.0 - 2025-07-24 + f"""# 1.0.0 - {datetime.today().strftime('%Y-%m-%d')} ## Added * Added Awesome feature @@ -178,7 +179,7 @@ def test_update_changelogs_for_release( ### `dev` * Added dependency `package2:0.2.0` - """ + """ ) @staticmethod @@ -194,7 +195,7 @@ def test_update_changelogs_for_release_with_no_dependencies( # versioned.md saved_text = changelogs.versioned_changelog_md.read_text() assert saved_text == cleandoc( - """# 1.0.0 - 2025-07-24 + f"""# 1.0.0 - {datetime.today().strftime('%Y-%m-%d')} ## Added * Added Awesome feature @@ -203,5 +204,5 @@ def test_update_changelogs_for_release_with_no_dependencies( ## Fixed * Fixed nasty bug - """ + """ ) From 91dd08896fa7d772a4b0253d3fcb3c75c5c0c7c1 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 09:50:48 +0200 Subject: [PATCH 13/33] Add types for track_changes.py --- exasol/toolbox/util/dependencies/track_changes.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_changes.py b/exasol/toolbox/util/dependencies/track_changes.py index 875b7cf6e..7bcc42bb0 100644 --- a/exasol/toolbox/util/dependencies/track_changes.py +++ b/exasol/toolbox/util/dependencies/track_changes.py @@ -10,7 +10,10 @@ ConfigDict, ) -from exasol.toolbox.util.dependencies.shared_models import Package +from exasol.toolbox.util.dependencies.shared_models import ( + NormalizedPackageStr, + Package, +) class DependencyChange(BaseModel): @@ -65,8 +68,8 @@ def from_package( class DependencyChanges(BaseModel): model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - old_dependencies: dict - current_dependencies: dict + old_dependencies: dict[NormalizedPackageStr, Package] + current_dependencies: dict[NormalizedPackageStr, Package] def _categorize_change(self, dependency_name: str) -> Union[DependencyChange, None]: """ From 7a9e6407c454f17d0dde3fcfb91aefc31eabec86 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 09:53:22 +0200 Subject: [PATCH 14/33] Rename old to previous for dependencies --- .../toolbox/util/dependencies/track_changes.py | 16 ++++++++-------- exasol/toolbox/util/release/changelog.py | 8 ++++---- test/unit/util/conftest.py | 2 +- .../unit/util/dependencies/track_changes_test.py | 14 ++++++++------ test/unit/util/release/changelog_test.py | 4 ++-- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_changes.py b/exasol/toolbox/util/dependencies/track_changes.py index 7bcc42bb0..65d8b5237 100644 --- a/exasol/toolbox/util/dependencies/track_changes.py +++ b/exasol/toolbox/util/dependencies/track_changes.py @@ -68,21 +68,21 @@ def from_package( class DependencyChanges(BaseModel): model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - old_dependencies: dict[NormalizedPackageStr, Package] + previous_dependencies: dict[NormalizedPackageStr, Package] current_dependencies: dict[NormalizedPackageStr, Package] def _categorize_change(self, dependency_name: str) -> Union[DependencyChange, None]: """ Categorize dependency change as removed, added, or updated. """ - old_dependency = self.old_dependencies.get(dependency_name) + previous_dependency = self.previous_dependencies.get(dependency_name) current_dependency = self.current_dependencies.get(dependency_name) - if old_dependency and not current_dependency: - return RemovedDependency.from_package(old_dependency) - elif not old_dependency and current_dependency: + if previous_dependency and not current_dependency: + return RemovedDependency.from_package(previous_dependency) + elif not previous_dependency and current_dependency: return AddedDependency.from_package(current_dependency) - elif old_dependency.version != current_dependency.version: # type: ignore - return UpdatedDependency.from_package(old_dependency, current_dependency) # type: ignore + elif previous_dependency.version != current_dependency.version: # type: ignore + return UpdatedDependency.from_package(previous_dependency, current_dependency) # type: ignore # dependency was unchanged between versions return None @@ -92,7 +92,7 @@ def changes(self) -> list[DependencyChange]: Return dependency changes """ all_dependencies = sorted( - self.old_dependencies.keys() | self.current_dependencies.keys() + self.previous_dependencies.keys() | self.current_dependencies.keys() ) return [ change_dependency diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index d22817f95..21ec7b427 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -57,7 +57,7 @@ def _extract_unreleased_notes(self) -> str: return unreleased_content def _prepare_dependency_update(self) -> str: - old_dependencies_in_groups = get_dependencies_from_latest_tag() + previous_dependencies_in_groups = get_dependencies_from_latest_tag() current_dependencies_in_groups = get_dependencies( working_directory=self.source_path ) @@ -67,16 +67,16 @@ def _prepare_dependency_update(self) -> str: groups = list( dict.fromkeys( chain( - old_dependencies_in_groups.keys(), + previous_dependencies_in_groups.keys(), current_dependencies_in_groups.keys(), ) ) ) for group in groups: - old_dependencies = old_dependencies_in_groups.get(group, {}) + previous_dependencies = previous_dependencies_in_groups.get(group, {}) current_dependencies = current_dependencies_in_groups.get(group, {}) changes = DependencyChanges( - old_dependencies=old_dependencies, + previous_dependencies=previous_dependencies, current_dependencies=current_dependencies, ).changes if changes: diff --git a/test/unit/util/conftest.py b/test/unit/util/conftest.py index d1d41a730..853123984 100644 --- a/test/unit/util/conftest.py +++ b/test/unit/util/conftest.py @@ -17,7 +17,7 @@ def dev_group(): @pytest.fixture(scope="module") -def old_dependencies(main_group, dev_group): +def previous_dependencies(main_group, dev_group): deps = OrderedDict() deps[main_group.name] = {"package1": Package(name="package1", version="0.0.1")} return deps diff --git a/test/unit/util/dependencies/track_changes_test.py b/test/unit/util/dependencies/track_changes_test.py index d0b8dc4ff..60f26b553 100644 --- a/test/unit/util/dependencies/track_changes_test.py +++ b/test/unit/util/dependencies/track_changes_test.py @@ -24,7 +24,8 @@ class TestDependencyChanges: @staticmethod def test_removed_dependency(): changes = DependencyChanges( - old_dependencies=SamplePackage().dependency_dict, current_dependencies={} + previous_dependencies=SamplePackage().dependency_dict, + current_dependencies={}, ) result = changes._categorize_change(SamplePackage.name) @@ -35,7 +36,8 @@ def test_removed_dependency(): @staticmethod def test_added_dependency(): changes = DependencyChanges( - old_dependencies={}, current_dependencies=SamplePackage().dependency_dict + previous_dependencies={}, + current_dependencies=SamplePackage().dependency_dict, ) result = changes._categorize_change(SamplePackage.name) @@ -48,7 +50,7 @@ def test_updated_dependency(): old_package = Package(name=SamplePackage.name, version="24.1.0") changes = DependencyChanges( - old_dependencies={SamplePackage.name: old_package}, + previous_dependencies={SamplePackage.name: old_package}, current_dependencies=SamplePackage().dependency_dict, ) @@ -62,7 +64,7 @@ def test_updated_dependency(): @staticmethod def test_dependency_without_changes(): changes = DependencyChanges( - old_dependencies=SamplePackage().dependency_dict, + previous_dependencies=SamplePackage().dependency_dict, current_dependencies=SamplePackage().dependency_dict, ) @@ -73,7 +75,7 @@ def test_dependency_without_changes(): @staticmethod def test_changes_with_no_changed_dependencies(): changes = DependencyChanges( - old_dependencies=SamplePackage().dependency_dict, + previous_dependencies=SamplePackage().dependency_dict, current_dependencies=SamplePackage().dependency_dict, ) @@ -86,7 +88,7 @@ def test_changes_with_changed_dependencies(): current_dependencies[added_package.name] = added_package changes = DependencyChanges( - old_dependencies=SamplePackage().dependency_dict, + previous_dependencies=SamplePackage().dependency_dict, current_dependencies=current_dependencies, ) diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index fac7064e5..ad33ff310 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -73,10 +73,10 @@ def unreleased_md(changelogs): @pytest.fixture(scope="function") -def mock_dependencies(dependencies, old_dependencies): +def mock_dependencies(dependencies, previous_dependencies): with mock.patch.multiple( "exasol.toolbox.util.release.changelog", - get_dependencies_from_latest_tag=lambda: old_dependencies, + get_dependencies_from_latest_tag=lambda: previous_dependencies, get_dependencies=lambda working_directory: dependencies, ): yield From 48013843f96921a3343f2478bd905da002e2c44a Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 09:57:18 +0200 Subject: [PATCH 15/33] Use group to be consistent with what we use in dependency changes and pyproject.toml --- exasol/toolbox/util/dependencies/licenses.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/exasol/toolbox/util/dependencies/licenses.py b/exasol/toolbox/util/dependencies/licenses.py index 059f8e7d8..4dcdb58d7 100644 --- a/exasol/toolbox/util/dependencies/licenses.py +++ b/exasol/toolbox/util/dependencies/licenses.py @@ -130,11 +130,15 @@ class PackageLicenseReport: @staticmethod def _format_group_table_header(group: str): - _group = "".join([word.capitalize() for word in group.strip().split()]) - text = f"## `{group}` Dependencies\n" - text += "|Package|Version|License|\n" - text += "|---|---|---|\n" - return text + return ( + cleandoc( + f"""## `{group}` Dependencies + |Package|Version|License| + |---|---|---| + """ + ) + + "\n" + ) def _format_group_table( self, group: str, group_package_names: set[NormalizedPackageStr] From 86b11af1da5e86ab4a344f04705e23e48161f8c1 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 09:58:14 +0200 Subject: [PATCH 16/33] Remove comment as not needed or useful --- exasol/toolbox/util/release/changelog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 21ec7b427..4b7db1c8e 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -80,7 +80,6 @@ def _prepare_dependency_update(self) -> str: current_dependencies=current_dependencies, ).changes if changes: - # nicer group heading content += f"\n### `{group}`\n" content += "\n".join(str(change) for change in changes) content += "\n" From d867d3c4a9e410dfca4ca81a115f031944eb4b7b Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 09:59:49 +0200 Subject: [PATCH 17/33] Switch source_path to be correctly named root_path --- exasol/toolbox/nox/_release.py | 2 +- exasol/toolbox/util/release/changelog.py | 6 +++--- test/unit/util/release/changelog_test.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/exasol/toolbox/nox/_release.py b/exasol/toolbox/nox/_release.py index 8236600e9..c3bf1a11e 100644 --- a/exasol/toolbox/nox/_release.py +++ b/exasol/toolbox/nox/_release.py @@ -120,7 +120,7 @@ def prepare_release(session: Session) -> None: changelogs = Changelogs( changes_path=PROJECT_CONFIG.doc / "changes", - source_path=PROJECT_CONFIG.root, + root_path=PROJECT_CONFIG.root, version=new_version, ) changelogs.update_changelogs_for_release() diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 4b7db1c8e..008e82c2a 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -16,12 +16,12 @@ class Changelogs: - def __init__(self, changes_path: Path, source_path: Path, version: Version) -> None: + def __init__(self, changes_path: Path, root_path: Path, version: Version) -> None: self.version = version self.unreleased_md: Path = changes_path / "unreleased.md" self.versioned_changelog_md: Path = changes_path / f"changes_{version}.md" self.changelog_md: Path = changes_path / "changelog.md" - self.source_path: Path = source_path + self.root_path: Path = root_path def _create_new_unreleased(self): """ @@ -59,7 +59,7 @@ def _extract_unreleased_notes(self) -> str: def _prepare_dependency_update(self) -> str: previous_dependencies_in_groups = get_dependencies_from_latest_tag() current_dependencies_in_groups = get_dependencies( - working_directory=self.source_path + working_directory=self.root_path ) content = "" diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index ad33ff310..ea8a3fcc6 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -98,7 +98,7 @@ def changelogs(tmp_path) -> Changelogs: changes_path.mkdir(parents=True) return Changelogs( changes_path=changes_path, - source_path=tmp_path, + root_path=tmp_path, version=Version(major=1, minor=0, patch=0), ) From 9023f1d0ede4ea5bd140e84dca95643d3bbf0cda Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:26:44 +0200 Subject: [PATCH 18/33] Add output types to methods --- exasol/toolbox/util/dependencies/licenses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol/toolbox/util/dependencies/licenses.py b/exasol/toolbox/util/dependencies/licenses.py index 4dcdb58d7..8eb47828a 100644 --- a/exasol/toolbox/util/dependencies/licenses.py +++ b/exasol/toolbox/util/dependencies/licenses.py @@ -129,7 +129,7 @@ class PackageLicenseReport: licenses: dict[NormalizedPackageStr, PackageLicense] @staticmethod - def _format_group_table_header(group: str): + def _format_group_table_header(group: str) -> str: return ( cleandoc( f"""## `{group}` Dependencies @@ -142,7 +142,7 @@ def _format_group_table_header(group: str): def _format_group_table( self, group: str, group_package_names: set[NormalizedPackageStr] - ): + ) -> str: group_header = self._format_group_table_header(group=group) rows_text = "" From 5265499f18334868de97bc1525579f6e055df069 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:31:20 +0200 Subject: [PATCH 19/33] Simplify output for _format_table_row --- exasol/toolbox/util/dependencies/licenses.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/exasol/toolbox/util/dependencies/licenses.py b/exasol/toolbox/util/dependencies/licenses.py index 8eb47828a..4a06340e0 100644 --- a/exasol/toolbox/util/dependencies/licenses.py +++ b/exasol/toolbox/util/dependencies/licenses.py @@ -154,23 +154,15 @@ def _format_group_table( @staticmethod def _format_table_row(license_info: PackageLicense) -> str: - text = "" - # column: package + row_package = f"{license_info.name}" if license_info.package_link: - text += f"|[{license_info.name}]({license_info.package_link})" - else: - text += f"|{license_info.name}" + row_package = f"[{license_info.name}]({license_info.package_link})" - # column: version - text += f"|{license_info.version}" - - # column: license + row_license = f"{license_info.license}" if license_info.license_link: - text += f"|[{license_info.license}]({license_info.license_link})|" - else: - text += f"|{license_info.license}|" + row_license = f"[{license_info.license}]({license_info.license_link})" - return text + "\n" + return f"|{row_package}|{license_info.version}|{row_license}|\n" def to_markdown(self) -> str: rows = "" From 5d75414318d302b166cc8d779b96398da327704c Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:33:00 +0200 Subject: [PATCH 20/33] Simplify output for _format_group_table_header --- exasol/toolbox/util/dependencies/licenses.py | 11 ++++------- test/unit/util/dependencies/licenses_test.py | 3 +-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/exasol/toolbox/util/dependencies/licenses.py b/exasol/toolbox/util/dependencies/licenses.py index 4a06340e0..2d7ca080f 100644 --- a/exasol/toolbox/util/dependencies/licenses.py +++ b/exasol/toolbox/util/dependencies/licenses.py @@ -130,14 +130,11 @@ class PackageLicenseReport: @staticmethod def _format_group_table_header(group: str) -> str: - return ( - cleandoc( - f"""## `{group}` Dependencies + return cleandoc( + f"""## `{group}` Dependencies |Package|Version|License| |---|---|---| - """ - ) - + "\n" + """ ) def _format_group_table( @@ -150,7 +147,7 @@ def _format_group_table( if license_info := self.licenses.get(package_name): rows_text += self._format_table_row(license_info=license_info) - return f"""{group_header}{rows_text}\n""" + return f"""{group_header}\n{rows_text}\n""" @staticmethod def _format_table_row(license_info: PackageLicense) -> str: diff --git a/test/unit/util/dependencies/licenses_test.py b/test/unit/util/dependencies/licenses_test.py index f207aaa39..2bdcaa025 100644 --- a/test/unit/util/dependencies/licenses_test.py +++ b/test/unit/util/dependencies/licenses_test.py @@ -122,8 +122,7 @@ def test_format_group_table_header(package_license_report, main_group): ) assert ( - result - == "## `main` Dependencies\n|Package|Version|License|\n|---|---|---|\n" + result == "## `main` Dependencies\n|Package|Version|License|\n|---|---|---|" ) @staticmethod From b7c311c7fce9a9b94d88228c6a642de163bdd964 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:36:49 +0200 Subject: [PATCH 21/33] Use normalized name from object instead of import --- exasol/toolbox/util/dependencies/licenses.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/exasol/toolbox/util/dependencies/licenses.py b/exasol/toolbox/util/dependencies/licenses.py index 2d7ca080f..ce1bf0143 100644 --- a/exasol/toolbox/util/dependencies/licenses.py +++ b/exasol/toolbox/util/dependencies/licenses.py @@ -13,7 +13,6 @@ from exasol.toolbox.util.dependencies.shared_models import ( NormalizedPackageStr, Package, - normalize_package_name, ) LICENSE_MAPPING_TO_ABBREVIATION = { @@ -97,13 +96,16 @@ def select_most_restrictive(licenses: list[str]) -> str: def _packages_from_json(json: str) -> dict[NormalizedPackageStr, PackageLicense]: packages = loads(json) return { - normalize_package_name(package["Name"]): PackageLicense( - name=package["Name"], - package_link=package["URL"], - version=package["Version"], - license=package["License"], - ) + package_license.normalized_name: package_license for package in packages + if ( + package_license := PackageLicense( + name=package["Name"], + package_link=package["URL"], + version=package["Version"], + license=package["License"], + ) + ) } From 5efc41716f9e10eef94367f42213059a0ffa3f9e Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:39:37 +0200 Subject: [PATCH 22/33] Switch from usage of string with += to using append in a list and joining --- exasol/toolbox/util/dependencies/licenses.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/exasol/toolbox/util/dependencies/licenses.py b/exasol/toolbox/util/dependencies/licenses.py index ce1bf0143..53e219682 100644 --- a/exasol/toolbox/util/dependencies/licenses.py +++ b/exasol/toolbox/util/dependencies/licenses.py @@ -144,12 +144,12 @@ def _format_group_table( ) -> str: group_header = self._format_group_table_header(group=group) - rows_text = "" + rows = [] for package_name in sorted(group_package_names): if license_info := self.licenses.get(package_name): - rows_text += self._format_table_row(license_info=license_info) + rows.append(self._format_table_row(license_info=license_info)) - return f"""{group_header}\n{rows_text}\n""" + return f"""{group_header}\n{''.join(rows)}\n""" @staticmethod def _format_table_row(license_info: PackageLicense) -> str: @@ -164,10 +164,12 @@ def _format_table_row(license_info: PackageLicense) -> str: return f"|{row_package}|{license_info.version}|{row_license}|\n" def to_markdown(self) -> str: - rows = "" + rows = [] for group in self.dependencies: group_package_names = set(self.dependencies[group].keys()) - rows += self._format_group_table( - group=group, group_package_names=group_package_names + rows.append( + self._format_group_table( + group=group, group_package_names=group_package_names + ) ) - return cleandoc(f"""# Dependencies\n\n{rows}""") + return cleandoc(f"""# Dependencies\n\n{''.join(rows)}""") From edf985a6eefe2898036d90c6c5d1e85448e3c717 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:41:50 +0200 Subject: [PATCH 23/33] Switch from old_version to previous_version --- exasol/toolbox/util/dependencies/track_changes.py | 6 +++--- test/unit/util/version_test.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_changes.py b/exasol/toolbox/util/dependencies/track_changes.py index 65d8b5237..28e99a1b4 100644 --- a/exasol/toolbox/util/dependencies/track_changes.py +++ b/exasol/toolbox/util/dependencies/track_changes.py @@ -45,12 +45,12 @@ def from_package(cls, package: Package) -> RemovedDependency: class UpdatedDependency(DependencyChange): - old_version: Version + previous_version: Version current_version: Version def __str__(self) -> str: return ( - f"* Updated dependency `{self.name}:{self.old_version}` " + f"* Updated dependency `{self.name}:{self.previous_version}` " f"to `{self.current_version}`" ) @@ -60,7 +60,7 @@ def from_package( ) -> UpdatedDependency: return cls( name=old_package.name, - old_version=old_package.version, + previous_version=old_package.version, current_version=current_package.version, ) diff --git a/test/unit/util/version_test.py b/test/unit/util/version_test.py index 9a2a4f326..96455a7c7 100644 --- a/test/unit/util/version_test.py +++ b/test/unit/util/version_test.py @@ -24,7 +24,7 @@ def test_create_version_from_string(input, expected): @pytest.mark.parametrize( - "old_version,new_version,expected", + "previous_version,new_version,expected", [ (Version(1, 2, 3), Version(1, 2, 4), True), (Version(1, 2, 3), Version(1, 3, 3), True), @@ -34,8 +34,8 @@ def test_create_version_from_string(input, expected): (Version(1, 2, 3), Version(0, 3, 3), False), ], ) -def test_is_later_version(old_version, new_version, expected): - actual = new_version > old_version +def test_is_later_version(previous_version, new_version, expected): + actual = new_version > previous_version assert expected == actual From d26f7aea03b77e81a3064fe67f4efba26715fc12 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:42:32 +0200 Subject: [PATCH 24/33] Switch from old_package to previous_package --- exasol/toolbox/util/dependencies/track_changes.py | 6 +++--- test/unit/util/dependencies/track_changes_test.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_changes.py b/exasol/toolbox/util/dependencies/track_changes.py index 28e99a1b4..5a97e5929 100644 --- a/exasol/toolbox/util/dependencies/track_changes.py +++ b/exasol/toolbox/util/dependencies/track_changes.py @@ -56,11 +56,11 @@ def __str__(self) -> str: @classmethod def from_package( - cls, old_package: Package, current_package: Package + cls, previous_package: Package, current_package: Package ) -> UpdatedDependency: return cls( - name=old_package.name, - previous_version=old_package.version, + name=previous_package.name, + previous_version=previous_package.version, current_version=current_package.version, ) diff --git a/test/unit/util/dependencies/track_changes_test.py b/test/unit/util/dependencies/track_changes_test.py index 60f26b553..bb1669466 100644 --- a/test/unit/util/dependencies/track_changes_test.py +++ b/test/unit/util/dependencies/track_changes_test.py @@ -47,17 +47,17 @@ def test_added_dependency(): @staticmethod def test_updated_dependency(): - old_package = Package(name=SamplePackage.name, version="24.1.0") + previous_package = Package(name=SamplePackage.name, version="24.1.0") changes = DependencyChanges( - previous_dependencies={SamplePackage.name: old_package}, + previous_dependencies={SamplePackage.name: previous_package}, current_dependencies=SamplePackage().dependency_dict, ) result = changes._categorize_change(SamplePackage.name) assert result == UpdatedDependency.from_package( - old_package=old_package, current_package=SamplePackage().package + previous_package=previous_package, current_package=SamplePackage().package ) assert str(result) == "* Updated dependency `black:24.1.0` to `25.1.0`" From d5069c4ca7bfcf6ac6cc53a253446e76514ad1ad Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:45:00 +0200 Subject: [PATCH 25/33] Switch from str to NormalizedPackageStr as that's what the dict keys are --- exasol/toolbox/util/dependencies/track_changes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/exasol/toolbox/util/dependencies/track_changes.py b/exasol/toolbox/util/dependencies/track_changes.py index 5a97e5929..298aa9029 100644 --- a/exasol/toolbox/util/dependencies/track_changes.py +++ b/exasol/toolbox/util/dependencies/track_changes.py @@ -71,7 +71,9 @@ class DependencyChanges(BaseModel): previous_dependencies: dict[NormalizedPackageStr, Package] current_dependencies: dict[NormalizedPackageStr, Package] - def _categorize_change(self, dependency_name: str) -> Union[DependencyChange, None]: + def _categorize_change( + self, dependency_name: NormalizedPackageStr + ) -> Union[DependencyChange, None]: """ Categorize dependency change as removed, added, or updated. """ From 22bc3daf5518de4720ef980490396bfdb7453273 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:48:03 +0200 Subject: [PATCH 26/33] Switch Union[*, None] with Optional[*] --- exasol/toolbox/nox/_artifacts.py | 7 ++----- exasol/toolbox/util/dependencies/track_changes.py | 10 +++++----- test/integration/project-template/nox_test.py | 11 +++++++---- test/unit/util/dependencies/licenses_test.py | 6 +++--- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/exasol/toolbox/nox/_artifacts.py b/exasol/toolbox/nox/_artifacts.py index 1bab4cbf5..92e2f667a 100644 --- a/exasol/toolbox/nox/_artifacts.py +++ b/exasol/toolbox/nox/_artifacts.py @@ -7,10 +7,7 @@ import sys from collections.abc import Iterable from pathlib import Path -from typing import ( - Optional, - Union, -) +from typing import Optional import nox from nox import Session @@ -188,7 +185,7 @@ def _copy_artifacts(source: Path, dest: Path, files: Iterable[str]): def _prepare_coverage_xml( - session: Session, source: Path, cwd: Union[Path, None] = None + session: Session, source: Path, cwd: Optional[Path] = None ) -> None: """ Prepare the coverage XML for input into Sonar diff --git a/exasol/toolbox/util/dependencies/track_changes.py b/exasol/toolbox/util/dependencies/track_changes.py index 298aa9029..5c4b77572 100644 --- a/exasol/toolbox/util/dependencies/track_changes.py +++ b/exasol/toolbox/util/dependencies/track_changes.py @@ -1,8 +1,6 @@ from __future__ import annotations -from typing import ( - Union, -) +from typing import Optional from packaging.version import Version from pydantic import ( @@ -73,7 +71,7 @@ class DependencyChanges(BaseModel): def _categorize_change( self, dependency_name: NormalizedPackageStr - ) -> Union[DependencyChange, None]: + ) -> Optional[DependencyChange]: """ Categorize dependency change as removed, added, or updated. """ @@ -84,7 +82,9 @@ def _categorize_change( elif not previous_dependency and current_dependency: return AddedDependency.from_package(current_dependency) elif previous_dependency.version != current_dependency.version: # type: ignore - return UpdatedDependency.from_package(previous_dependency, current_dependency) # type: ignore + return UpdatedDependency.from_package( + previous_dependency, current_dependency + ) # type: ignore # dependency was unchanged between versions return None diff --git a/test/integration/project-template/nox_test.py b/test/integration/project-template/nox_test.py index 36895901d..d28852ee6 100644 --- a/test/integration/project-template/nox_test.py +++ b/test/integration/project-template/nox_test.py @@ -1,4 +1,4 @@ -from typing import Union +from typing import Optional class TestSpecificNoxTasks: @@ -14,8 +14,10 @@ class TestSpecificNoxTasks: both the relationships between nox tasks and ensuring that the project-template passes CI tests when a new project is created from it. """ + @staticmethod - def _command(poetry_path: str, task: str, add_ons: Union[list[str], None]=None) -> list[str]: + def _command(poetry_path: str, task: str, + add_ons: Optional[list[str]] = None) -> list[str]: base = [poetry_path, "run", "--", "nox", "-s", task] if add_ons: base = base + ["--"] + add_ons @@ -33,9 +35,10 @@ def test_artifact_validate(self, poetry_path, run_command): run_command(lint_code) lint_security = self._command(poetry_path, "lint:security") run_command(lint_security) - test_unit = self._command(poetry_path, "test:unit",["--coverage"]) + test_unit = self._command(poetry_path, "test:unit", ["--coverage"]) run_command(test_unit) - test_integration = self._command(poetry_path, "test:integration",["--coverage"]) + test_integration = self._command(poetry_path, "test:integration", + ["--coverage"]) run_command(test_integration) # `artifacts:copy` is skipped here. This step has the pre-requisite that files # were uploaded to & then downloaded from the GitHub run's artifacts. diff --git a/test/unit/util/dependencies/licenses_test.py b/test/unit/util/dependencies/licenses_test.py index 2bdcaa025..a20222fab 100644 --- a/test/unit/util/dependencies/licenses_test.py +++ b/test/unit/util/dependencies/licenses_test.py @@ -1,4 +1,4 @@ -from typing import Union +from typing import Optional import pytest @@ -167,8 +167,8 @@ def test_format_group_table(package_license_report, dependencies, main_group): ) def test_format_table_row( package_license_report, - package_link: Union[str, None], - license_link: Union[str, None], + package_link: Optional[str], + license_link: Optional[str], expected, ): _license = PackageLicense( From c339a5f7168f2e2bcf31e56e901b8c2f0c8787e5 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:49:50 +0200 Subject: [PATCH 27/33] Add comment to make clearer what the operation does --- exasol/toolbox/util/dependencies/track_changes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exasol/toolbox/util/dependencies/track_changes.py b/exasol/toolbox/util/dependencies/track_changes.py index 5c4b77572..d9fae692e 100644 --- a/exasol/toolbox/util/dependencies/track_changes.py +++ b/exasol/toolbox/util/dependencies/track_changes.py @@ -93,6 +93,7 @@ def changes(self) -> list[DependencyChange]: """ Return dependency changes """ + # dict.keys() returns a set converted into a list by `sorted()` all_dependencies = sorted( self.previous_dependencies.keys() | self.current_dependencies.keys() ) From 96b98f8daef9c1cdb3f05a89318ffb6a05a9fea3 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 10:59:55 +0200 Subject: [PATCH 28/33] Add on description of Changelogs arguments --- exasol/toolbox/util/release/changelog.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 008e82c2a..9da87f639 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -17,6 +17,14 @@ class Changelogs: def __init__(self, changes_path: Path, root_path: Path, version: Version) -> None: + """ + Args: + changes_path: directory containing the changelog, e.g. `doc/changes/` + root_path: root directory of the current project, containing file + `pyproject.toml` + version: the version to be used in the `changes_{version}.md` and listed in the `changelog.md`. + """ + self.version = version self.unreleased_md: Path = changes_path / "unreleased.md" self.versioned_changelog_md: Path = changes_path / f"changes_{version}.md" From cfae78552a8aa76df88f6224b0b1f73979946ad0 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 12:04:57 +0200 Subject: [PATCH 29/33] Improve terminology & adapt _describe_dependency_changes to be simpler --- exasol/toolbox/util/release/changelog.py | 31 +++++++++++++----------- test/unit/util/release/changelog_test.py | 4 +-- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 9da87f639..8e34bd31c 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -19,10 +19,11 @@ class Changelogs: def __init__(self, changes_path: Path, root_path: Path, version: Version) -> None: """ Args: - changes_path: directory containing the changelog, e.g. `doc/changes/` + changes_path: directory containing the changelog & changes files, e.g. `doc/changes/` root_path: root directory of the current project, containing file `pyproject.toml` - version: the version to be used in the `changes_{version}.md` and listed in the `changelog.md`. + version: the version to be used in the versioned changes file and listed in + the `changelog.md`, which contains the index of the change log """ self.version = version @@ -39,23 +40,22 @@ def _create_new_unreleased(self): def _create_versioned_changelog(self, unreleased_content: str) -> None: """ - Create a changelog entry for a specific version. + Create a versioned changes file. Args: - unreleased_content: The content of the changelog entry. - + unreleased_content: the content of the (not yet versioned) changes """ template = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" template += f"\n{unreleased_content}" - if dependency_content := self._prepare_dependency_update(): + if dependency_content := self._describe_dependency_changes(): template += f"\n## Dependency Updates\n{dependency_content}" self.versioned_changelog_md.write_text(cleandoc(template)) def _extract_unreleased_notes(self) -> str: """ - Extract release notes from `unreleased.md`. + Extract (not yet versioned) changes from `unreleased.md`. """ with self.unreleased_md.open(mode="r", encoding="utf-8") as f: # skip header when reading in file, as contains # Unreleased @@ -64,13 +64,17 @@ def _extract_unreleased_notes(self) -> str: unreleased_content += "\n" return unreleased_content - def _prepare_dependency_update(self) -> str: + def _describe_dependency_changes(self) -> str: + """ + Describe the dependency changes between the latest tag and the current version + for use in the versioned changes file. + """ previous_dependencies_in_groups = get_dependencies_from_latest_tag() current_dependencies_in_groups = get_dependencies( working_directory=self.root_path ) - content = "" + changes_by_group: list[str] = [] # preserve order of keys from old group groups = list( dict.fromkeys( @@ -88,10 +92,9 @@ def _prepare_dependency_update(self) -> str: current_dependencies=current_dependencies, ).changes if changes: - content += f"\n### `{group}`\n" - content += "\n".join(str(change) for change in changes) - content += "\n" - return content + changes_str = "\n".join(str(change) for change in changes) + changes_by_group.append(f"\n### `{group}`\n{changes_str}\n") + return "".join(changes_by_group) def _update_changelog_table_of_contents(self) -> None: """ @@ -119,7 +122,7 @@ def update_changelogs_for_release(self) -> None: """ Rotates the changelogs as is needed for a release. - 1. Moves the contents of the `unreleased.md` to the `changes_.md` + 1. Moves the changes from the `unreleased.md` to the `changes_.md` 2. Create a new file `unreleased.md` 3. Updates the table of contents in the `changelog.md` with the new `changes_.md` """ diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index ea8a3fcc6..87475a284 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -132,8 +132,8 @@ def test_extract_unreleased_notes(changelogs, unreleased_md): assert result == SampleContent.changelog + "\n" @staticmethod - def test_prepare_dependency_update(changelogs, mock_dependencies): - result = changelogs._prepare_dependency_update() + def test_describe_dependency_changes(changelogs, mock_dependencies): + result = changelogs._describe_dependency_changes() assert result == ( "\n" "### `main`\n" From 31b48bc9f78ffabfc40278580096214e1d3cdd1c Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 13:00:14 +0200 Subject: [PATCH 30/33] Add deterministic sorting to versioned changes file --- exasol/toolbox/util/release/changelog.py | 30 ++++++++++++++++-------- test/unit/util/release/changelog_test.py | 16 +++++++++++++ 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 8e34bd31c..0209578b2 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -2,7 +2,6 @@ from datetime import datetime from inspect import cleandoc -from itertools import chain from pathlib import Path from exasol.toolbox.util.dependencies.poetry_dependencies import ( @@ -75,16 +74,12 @@ def _describe_dependency_changes(self) -> str: ) changes_by_group: list[str] = [] - # preserve order of keys from old group - groups = list( - dict.fromkeys( - chain( - previous_dependencies_in_groups.keys(), - current_dependencies_in_groups.keys(), - ) - ) + # dict.keys() returns a set + all_groups = ( + previous_dependencies_in_groups.keys() + | current_dependencies_in_groups.keys() ) - for group in groups: + for group in self._sort_groups(all_groups): previous_dependencies = previous_dependencies_in_groups.get(group, {}) current_dependencies = current_dependencies_in_groups.get(group, {}) changes = DependencyChanges( @@ -96,6 +91,21 @@ def _describe_dependency_changes(self) -> str: changes_by_group.append(f"\n### `{group}`\n{changes_str}\n") return "".join(changes_by_group) + @staticmethod + def _sort_groups(groups: set[str]) -> list[str]: + """ + Prepare a deterministic sorting for groups shown in the versioned changes file: + - `main` group should always be first + - remaining groups are sorted alphabetically + """ + main = "main" + if main not in groups: + # sorted converts set to list + return sorted(groups) + remaining_groups = groups - {main} + # sorted converts set to list + return [main] + sorted(remaining_groups) + def _update_changelog_table_of_contents(self) -> None: """ Read in existing `changelog.md` and append to appropriate sections diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index 87475a284..1c0002b60 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -143,6 +143,22 @@ def test_describe_dependency_changes(changelogs, mock_dependencies): "* Added dependency `package2:0.2.0`\n" ) + @staticmethod + @pytest.mark.parametrize( + "groups,expected", + [ + pytest.param( + {"dev", "abcd", "main"}, ["main", "abcd", "dev"], id="with_main" + ), + pytest.param( + {"dev", "abcd", "bacd"}, ["abcd", "bacd", "dev"], id="without_main" + ), + ], + ) + def test_sort_groups(changelogs, groups, expected): + result = changelogs._sort_groups(groups) + assert result == expected + @staticmethod def test_update_changelog_table_of_contents(changelogs, changes_md): changelogs._update_changelog_table_of_contents() From 898e10c23e134f35bb32ac894115f360daede9c1 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 13:08:56 +0200 Subject: [PATCH 31/33] Simplify _create_versioned_changelog and fix moved type --- exasol/toolbox/util/dependencies/track_changes.py | 4 ++-- exasol/toolbox/util/release/changelog.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_changes.py b/exasol/toolbox/util/dependencies/track_changes.py index d9fae692e..a684252a2 100644 --- a/exasol/toolbox/util/dependencies/track_changes.py +++ b/exasol/toolbox/util/dependencies/track_changes.py @@ -83,8 +83,8 @@ def _categorize_change( return AddedDependency.from_package(current_dependency) elif previous_dependency.version != current_dependency.version: # type: ignore return UpdatedDependency.from_package( - previous_dependency, current_dependency - ) # type: ignore + previous_dependency, current_dependency # type: ignore + ) # dependency was unchanged between versions return None diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 0209578b2..c8724d9ad 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -44,12 +44,13 @@ def _create_versioned_changelog(self, unreleased_content: str) -> None: Args: unreleased_content: the content of the (not yet versioned) changes """ - template = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" - template += f"\n{unreleased_content}" + header = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" - if dependency_content := self._describe_dependency_changes(): - template += f"\n## Dependency Updates\n{dependency_content}" + dependency_content = "" + if dependency_changes := self._describe_dependency_changes(): + dependency_content = f"## Dependency Updates\n{dependency_changes}" + template = cleandoc(f"{header}\n{unreleased_content}\n{dependency_content}") self.versioned_changelog_md.write_text(cleandoc(template)) def _extract_unreleased_notes(self) -> str: @@ -60,8 +61,7 @@ def _extract_unreleased_notes(self) -> str: # skip header when reading in file, as contains # Unreleased lines = f.readlines()[1:] unreleased_content = cleandoc("".join(lines)) - unreleased_content += "\n" - return unreleased_content + return unreleased_content + "\n" def _describe_dependency_changes(self) -> str: """ From b011f5f5e79eecdb9a60c198dead69da802ba9b3 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 13:17:26 +0200 Subject: [PATCH 32/33] Remove unneeded, second cleancode() usage --- exasol/toolbox/util/release/changelog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index c8724d9ad..89e7d6dc5 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -51,7 +51,7 @@ def _create_versioned_changelog(self, unreleased_content: str) -> None: dependency_content = f"## Dependency Updates\n{dependency_changes}" template = cleandoc(f"{header}\n{unreleased_content}\n{dependency_content}") - self.versioned_changelog_md.write_text(cleandoc(template)) + self.versioned_changelog_md.write_text(template) def _extract_unreleased_notes(self) -> str: """ From 3524c4524174dd0726ace131600e4320b042a322 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Fri, 25 Jul 2025 13:49:01 +0200 Subject: [PATCH 33/33] Switch text from changes to contents --- exasol/toolbox/util/release/changelog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 89e7d6dc5..0c248730f 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -132,7 +132,7 @@ def update_changelogs_for_release(self) -> None: """ Rotates the changelogs as is needed for a release. - 1. Moves the changes from the `unreleased.md` to the `changes_.md` + 1. Moves the contents from the `unreleased.md` to the `changes_.md` 2. Create a new file `unreleased.md` 3. Updates the table of contents in the `changelog.md` with the new `changes_.md` """