From b2090ae5881eff7324e22ce3bcf17e7e13699583 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 16 Sep 2025 00:08:02 +0000 Subject: [PATCH 1/6] chore: verify library distribution name --- .generator/cli.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.generator/cli.py b/.generator/cli.py index 02563c3a5a57..df88963c6e1a 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -26,6 +26,9 @@ from datetime import datetime from pathlib import Path from typing import Dict, List +from distutils.core import run_setup + +import tomli try: import synthtool @@ -546,12 +549,48 @@ def _verify_library_namespace(library_id: str, repo: str): ) +def _get_setup_dist_name(library_id: str, repo: str): + try: + dist = run_setup(f"{repo}/packages/{library_id}/setup.py") + return dist.get_name() + except Exception as e: + return None + + +def _get_toml_dist_name(library_id: str, repo: str): + try: + pyproject_toml_file = Path(f"{repo}/packages/{library_id}/pyproject.toml") + with open(pyproject_toml_file, "rb") as f: + data = tomli.load(f) + return data.get("project", {}).get("name") + except Exception as e: + return None + + +def _verify_library_dist_name(library_id: str, repo: str): + setup_dist_name = _get_setup_dist_name(library_id, repo) + toml_dist_name = _get_toml_dist_name(library_id, repo) + if setup_dist_name is None and toml_dist_name is None: + raise ValueError( + f"No valid `setup.py` or `pyproject.toml found for `{library_id}`." + ) + if setup_dist_name is not None and setup_dist_name != library_id: + raise ValueError( + f"The distribution name `{setup_dist_name} in `setup.py` does not match folder `{library_id}`" + ) + if toml_dist_name is not None and toml_dist_name != library_id: + raise ValueError( + f"The distribution name `{toml_dist_name} in `pyproject.toml` does not match folder `{library_id}`" + ) + + def handle_build(librarian: str = LIBRARIAN_DIR, repo: str = REPO_DIR): """The main coordinator for validating client library generation.""" try: request_data = _read_json_file(f"{librarian}/{BUILD_REQUEST_FILE}") library_id = _get_library_id(request_data) _verify_library_namespace(library_id, repo) + _verify_library_dist_name(library_id, repo) _run_nox_sessions(library_id, repo) except Exception as e: raise ValueError("Build failed.") from e From f0c0b17ceb373e6c78833e2237790432fc409d64 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 16 Sep 2025 09:17:00 +0000 Subject: [PATCH 2/6] add test coverage --- .generator/cli.py | 59 +++++++++++++++++++++++++-- .generator/requirements-test.in | 2 + .generator/test_cli.py | 70 ++++++++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 4 deletions(-) diff --git a/.generator/cli.py b/.generator/cli.py index df88963c6e1a..4e1cf35dcd61 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -25,7 +25,7 @@ import yaml from datetime import datetime from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional from distutils.core import run_setup import tomli @@ -549,25 +549,78 @@ def _verify_library_namespace(library_id: str, repo: str): ) -def _get_setup_dist_name(library_id: str, repo: str): +def _get_setup_dist_name(library_id: str, repo: str) -> Optional[str]: + """Gets the library distribution name from a `setup.py` file. + + Tries to execute `run_setup` on the file and extract the 'name' + property. Logs and returns None on any exception. + + Args: + library_id(str): id of the library. + repo(str): This directory will contain all directories that make up a + library, the .librarian folder, and any global file declared in + the config.yaml. + + Returns: + Optional[str]: The library distribution name if found and parsed, otherwise None. + """ try: dist = run_setup(f"{repo}/packages/{library_id}/setup.py") return dist.get_name() except Exception as e: + logger.debug( + f"failed to get distribution name for `{library_id}` from `setup.py` : {e}", + exc_info=True, + ) return None -def _get_toml_dist_name(library_id: str, repo: str): +def _get_toml_dist_name(library_id: str, repo: str) -> Optional[str]: + """Gets the library distribution name from a `pyproject.toml` file. + + Parses the TOML file and safely accesses the name from the + [project] table using `.get()`. Logs and returns None on any + exception (e.g., FileNotFoundError, TOMLDecodeError). + + Args: + library_id(str): id of the library. + repo(str): This directory will contain all directories that make up a + library, the .librarian folder, and any global file declared in + the config.yaml. + + Returns: + Optional[str]: The library distribution name if found and parsed, otherwise None. + """ try: pyproject_toml_file = Path(f"{repo}/packages/{library_id}/pyproject.toml") with open(pyproject_toml_file, "rb") as f: data = tomli.load(f) return data.get("project", {}).get("name") except Exception as e: + logger.debug( + f"failed to get distribution name for `{library_id}` from `pyproject.toml` : {e}", + exc_info=True, + ) return None def _verify_library_dist_name(library_id: str, repo: str): + """Verifies the library distribution name against its config files. + + This function ensures that: + 1. At least one of `setup.py` or `pyproject.toml` exists and is valid. + 2. Any existing config file's 'name' property matches the `library_id`. + + Args: + library_id: id of the library. + repo: This directory will contain all directories that make up a + library, the .librarian folder, and any global file declared in + the config.yaml. + + Raises: + ValueError: If no valid config file is found, or if a name + in an existing file does not match the `library_id`. + """ setup_dist_name = _get_setup_dist_name(library_id, repo) toml_dist_name = _get_toml_dist_name(library_id, repo) if setup_dist_name is None and toml_dist_name is None: diff --git a/.generator/requirements-test.in b/.generator/requirements-test.in index 7b852b04129f..699a125fd265 100644 --- a/.generator/requirements-test.in +++ b/.generator/requirements-test.in @@ -17,3 +17,5 @@ pytest-cov pytest-mock gcp-synthtool @ git+https://github.com/googleapis/synthtool@5aa438a342707842d11fbbb302c6277fbf9e4655 starlark-pyo3>=2025.1 +# TODO(https://github.com/googleapis/google-cloud-python/issues/14443): tomli is part of the standard library for Python 3.11. Remove once we use 3.11+ for librarian. +tomli diff --git a/.generator/test_cli.py b/.generator/test_cli.py index 7ebe90e93a5d..9c8438dbccfd 100644 --- a/.generator/test_cli.py +++ b/.generator/test_cli.py @@ -41,6 +41,8 @@ _get_library_id, _get_libraries_to_prepare_for_release, _get_previous_version, + _get_setup_dist_name, + _get_toml_dist_name, _locate_and_extract_artifact, _process_changelog, _process_version_file, @@ -52,6 +54,7 @@ _update_changelog_for_library, _update_global_changelog, _update_version_for_library, + _verify_library_dist_name, _verify_library_namespace, _write_json_file, _write_text_file, @@ -525,7 +528,8 @@ def test_handle_build_success(caplog, mocker, mock_build_request_file): caplog.set_level(logging.INFO) mocker.patch("cli._run_nox_sessions") - mocker.patch("cli._verify_library_namespace", return_value=True) + mocker.patch("cli._verify_library_namespace") + mocker.patch("cli._verify_library_dist_name") handle_build() assert "'build' command executed." in caplog.text @@ -922,6 +926,70 @@ def test_determine_library_namespace_fails_not_subpath(): _determine_library_namespace(gapic_parent_path, pkg_root_path) +def test_get_setup_dist_name_exists(mocker): + """Tests that a valid library distribution name exists in `pyproject.toml`.""" + mock_dist = MagicMock() + mock_dist.get_name.return_value = "my-lib" + mocker.patch("cli.run_setup", return_value=mock_dist) + assert _get_setup_dist_name("my-lib", "repo") == "my-lib" + + +def test_get_setup_dist_name_file_not_found(caplog): + """Tests that distribution name is None if `setup.py` does not exist.""" + caplog.set_level(logging.DEBUG) + assert _get_setup_dist_name("my-lib", "repo") is None + assert len(caplog.records) == 1 + + +def test_get_toml_dist_name_exists(mocker): + """Tests that a valid library distribution name exists in `pyproject.toml`.""" + mock_data = {"project": {"name": "my-lib"}} + mocker.patch("tomli.load", return_value=mock_data) + mocker.patch("builtins.open", mocker.mock_open(read_data=b"fake toml data")) + assert _get_toml_dist_name("my-lib", "repo") == "my-lib" + + +def test_get_toml_dist_name_file_not_found(caplog): + """Tests that distribution name is None if `pyproject.toml` does not exist.""" + caplog.set_level(logging.DEBUG) + assert _get_toml_dist_name("my-lib", "repo") is None + assert len(caplog.records) == 1 + + +def test_verify_library_dist_name_setup_success(mocker): + """Tests success when a library distribution name in setup.py is valid.""" + mock_setup_file = mocker.patch("cli._get_setup_dist_name", return_value="my-lib") + _verify_library_dist_name("my-lib", "repo") + mock_setup_file.assert_called_once_with("my-lib", "repo") + + +def test_verify_library_dist_name_setup_success(mocker): + """Tests success when a library distribution name in toml is valid.""" + mock_toml_file = mocker.patch("cli._get_toml_dist_name", return_value="my-lib") + _verify_library_dist_name("my-lib", "repo") + mock_toml_file.assert_called_once_with("my-lib", "repo") + + +def test_verify_library_dist_name_fail(): + """Tests failure when a library does not have a `setup.py` or `pyproject.toml`.""" + with pytest.raises(ValueError): + _verify_library_dist_name("my-lib", "repo") + + +def test_verify_library_dist_name_setup_fail(mocker): + """Tests failure when a library has an invalid distribution name in `setup.py`.""" + mocker.patch("cli._get_setup_dist_name", return_value="invalid-lib-name") + with pytest.raises(ValueError): + _verify_library_dist_name("my-lib", "repo") + + +def test_verify_library_dist_name_toml_fail(mocker): + """Tests failure when a library has an invalid distribution name in `pyproject.toml`.""" + mocker.patch("cli._get_toml_dist_name", return_value="invalid-lib-name") + with pytest.raises(ValueError): + _verify_library_dist_name("my-lib", "repo") + + def test_verify_library_namespace_success_valid(mocker, mock_path_class): """Tests success when a single valid namespace is found.""" # 1. Get the mock instance from the mock class's return_value From afc72062e3c2c9e4e4d14492a81cad17b1dc13e0 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 16 Sep 2025 10:24:02 +0000 Subject: [PATCH 3/6] use build for dist name --- .generator/cli.py | 76 ++++++--------------------------- .generator/requirements-test.in | 2 - .generator/test_cli.py | 63 ++++----------------------- 3 files changed, 23 insertions(+), 118 deletions(-) diff --git a/.generator/cli.py b/.generator/cli.py index 4e1cf35dcd61..17ba6602465c 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -25,10 +25,9 @@ import yaml from datetime import datetime from pathlib import Path -from typing import Dict, List, Optional -from distutils.core import run_setup +from typing import Dict, List -import tomli +import build.util try: import synthtool @@ -549,59 +548,22 @@ def _verify_library_namespace(library_id: str, repo: str): ) -def _get_setup_dist_name(library_id: str, repo: str) -> Optional[str]: - """Gets the library distribution name from a `setup.py` file. - - Tries to execute `run_setup` on the file and extract the 'name' - property. Logs and returns None on any exception. - - Args: - library_id(str): id of the library. - repo(str): This directory will contain all directories that make up a - library, the .librarian folder, and any global file declared in - the config.yaml. - - Returns: - Optional[str]: The library distribution name if found and parsed, otherwise None. +def _get_library_dist_name(library_id: str, repo: str) -> str: """ - try: - dist = run_setup(f"{repo}/packages/{library_id}/setup.py") - return dist.get_name() - except Exception as e: - logger.debug( - f"failed to get distribution name for `{library_id}` from `setup.py` : {e}", - exc_info=True, - ) - return None - - -def _get_toml_dist_name(library_id: str, repo: str) -> Optional[str]: - """Gets the library distribution name from a `pyproject.toml` file. - - Parses the TOML file and safely accesses the name from the - [project] table using `.get()`. Logs and returns None on any - exception (e.g., FileNotFoundError, TOMLDecodeError). + Gets the package name by programmatically building the metadata. Args: - library_id(str): id of the library. - repo(str): This directory will contain all directories that make up a + library_id: id of the library. + repo: This directory will contain all directories that make up a library, the .librarian folder, and any global file declared in the config.yaml. Returns: - Optional[str]: The library distribution name if found and parsed, otherwise None. + str: The library name string if found, otherwise None. """ - try: - pyproject_toml_file = Path(f"{repo}/packages/{library_id}/pyproject.toml") - with open(pyproject_toml_file, "rb") as f: - data = tomli.load(f) - return data.get("project", {}).get("name") - except Exception as e: - logger.debug( - f"failed to get distribution name for `{library_id}` from `pyproject.toml` : {e}", - exc_info=True, - ) - return None + library_path = f"{repo}/packages/{library_id}" + metadata = build.util.project_wheel_metadata(library_path) + return metadata.get("name") def _verify_library_dist_name(library_id: str, repo: str): @@ -618,22 +580,12 @@ def _verify_library_dist_name(library_id: str, repo: str): the config.yaml. Raises: - ValueError: If no valid config file is found, or if a name - in an existing file does not match the `library_id`. + ValueError: If a name in an existing config file does not match the `library_id`. """ - setup_dist_name = _get_setup_dist_name(library_id, repo) - toml_dist_name = _get_toml_dist_name(library_id, repo) - if setup_dist_name is None and toml_dist_name is None: - raise ValueError( - f"No valid `setup.py` or `pyproject.toml found for `{library_id}`." - ) - if setup_dist_name is not None and setup_dist_name != library_id: - raise ValueError( - f"The distribution name `{setup_dist_name} in `setup.py` does not match folder `{library_id}`" - ) - if toml_dist_name is not None and toml_dist_name != library_id: + dist_name = _get_library_dist_name(library_id, repo) + if dist_name != library_id: raise ValueError( - f"The distribution name `{toml_dist_name} in `pyproject.toml` does not match folder `{library_id}`" + f"The distribution name `{dist_name}` does not match the folder `{library_id}`." ) diff --git a/.generator/requirements-test.in b/.generator/requirements-test.in index 699a125fd265..7b852b04129f 100644 --- a/.generator/requirements-test.in +++ b/.generator/requirements-test.in @@ -17,5 +17,3 @@ pytest-cov pytest-mock gcp-synthtool @ git+https://github.com/googleapis/synthtool@5aa438a342707842d11fbbb302c6277fbf9e4655 starlark-pyo3>=2025.1 -# TODO(https://github.com/googleapis/google-cloud-python/issues/14443): tomli is part of the standard library for Python 3.11. Remove once we use 3.11+ for librarian. -tomli diff --git a/.generator/test_cli.py b/.generator/test_cli.py index 9c8438dbccfd..dcac0df5a0c3 100644 --- a/.generator/test_cli.py +++ b/.generator/test_cli.py @@ -37,12 +37,11 @@ _copy_files_needed_for_post_processing, _create_main_version_header, _determine_bazel_rule, + _get_library_dist_name, _determine_library_namespace, _get_library_id, _get_libraries_to_prepare_for_release, _get_previous_version, - _get_setup_dist_name, - _get_toml_dist_name, _locate_and_extract_artifact, _process_changelog, _process_version_file, @@ -926,66 +925,22 @@ def test_determine_library_namespace_fails_not_subpath(): _determine_library_namespace(gapic_parent_path, pkg_root_path) -def test_get_setup_dist_name_exists(mocker): - """Tests that a valid library distribution name exists in `pyproject.toml`.""" - mock_dist = MagicMock() - mock_dist.get_name.return_value = "my-lib" - mocker.patch("cli.run_setup", return_value=mock_dist) - assert _get_setup_dist_name("my-lib", "repo") == "my-lib" - - -def test_get_setup_dist_name_file_not_found(caplog): - """Tests that distribution name is None if `setup.py` does not exist.""" - caplog.set_level(logging.DEBUG) - assert _get_setup_dist_name("my-lib", "repo") is None - assert len(caplog.records) == 1 - - -def test_get_toml_dist_name_exists(mocker): - """Tests that a valid library distribution name exists in `pyproject.toml`.""" - mock_data = {"project": {"name": "my-lib"}} - mocker.patch("tomli.load", return_value=mock_data) - mocker.patch("builtins.open", mocker.mock_open(read_data=b"fake toml data")) - assert _get_toml_dist_name("my-lib", "repo") == "my-lib" - - -def test_get_toml_dist_name_file_not_found(caplog): - """Tests that distribution name is None if `pyproject.toml` does not exist.""" - caplog.set_level(logging.DEBUG) - assert _get_toml_dist_name("my-lib", "repo") is None - assert len(caplog.records) == 1 +def test_get_library_dist_name_success(mocker): + mock_metadata = {"name": "my-lib", "version": "1.0.0"} + mocker.patch("build.util.project_wheel_metadata", return_value=mock_metadata) + assert _get_library_dist_name("my-lib", "repo") == "my-lib" def test_verify_library_dist_name_setup_success(mocker): """Tests success when a library distribution name in setup.py is valid.""" - mock_setup_file = mocker.patch("cli._get_setup_dist_name", return_value="my-lib") + mock_setup_file = mocker.patch("cli._get_library_dist_name", return_value="my-lib") _verify_library_dist_name("my-lib", "repo") mock_setup_file.assert_called_once_with("my-lib", "repo") -def test_verify_library_dist_name_setup_success(mocker): - """Tests success when a library distribution name in toml is valid.""" - mock_toml_file = mocker.patch("cli._get_toml_dist_name", return_value="my-lib") - _verify_library_dist_name("my-lib", "repo") - mock_toml_file.assert_called_once_with("my-lib", "repo") - - -def test_verify_library_dist_name_fail(): - """Tests failure when a library does not have a `setup.py` or `pyproject.toml`.""" - with pytest.raises(ValueError): - _verify_library_dist_name("my-lib", "repo") - - -def test_verify_library_dist_name_setup_fail(mocker): - """Tests failure when a library has an invalid distribution name in `setup.py`.""" - mocker.patch("cli._get_setup_dist_name", return_value="invalid-lib-name") - with pytest.raises(ValueError): - _verify_library_dist_name("my-lib", "repo") - - -def test_verify_library_dist_name_toml_fail(mocker): - """Tests failure when a library has an invalid distribution name in `pyproject.toml`.""" - mocker.patch("cli._get_toml_dist_name", return_value="invalid-lib-name") +def test_verify_library_dist_name_fail(mocker): + """Tests failure when a library-id does not match the libary distribution name.""" + mocker.patch("cli._get_library_dist_name", return_value="invalid-lib") with pytest.raises(ValueError): _verify_library_dist_name("my-lib", "repo") From edd6ef3f4cf9ca0b89c04d6c409e20868c7cde70 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 16 Sep 2025 10:25:47 +0000 Subject: [PATCH 4/6] install build --- .generator/requirements-test.in | 1 + 1 file changed, 1 insertion(+) diff --git a/.generator/requirements-test.in b/.generator/requirements-test.in index 7b852b04129f..b769b9b6d7b4 100644 --- a/.generator/requirements-test.in +++ b/.generator/requirements-test.in @@ -17,3 +17,4 @@ pytest-cov pytest-mock gcp-synthtool @ git+https://github.com/googleapis/synthtool@5aa438a342707842d11fbbb302c6277fbf9e4655 starlark-pyo3>=2025.1 +build \ No newline at end of file From d7cff01a1fc8ffb35e0dd89df5f6c4f3c60499ed Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 16 Sep 2025 17:27:05 +0000 Subject: [PATCH 5/6] install requirements --- .generator/Dockerfile | 4 ++++ .generator/requirements.in | 1 + 2 files changed, 5 insertions(+) create mode 100644 .generator/requirements.in diff --git a/.generator/Dockerfile b/.generator/Dockerfile index b80af6f7103a..b1f49582c2c3 100644 --- a/.generator/Dockerfile +++ b/.generator/Dockerfile @@ -172,6 +172,10 @@ RUN git clone --depth 1 https://github.com/googleapis/synthtool.git /tmp/synthto bazel_env/bin/python3.9 -m pip install /tmp/synthtool nox && \ rm -rf /tmp/synthtool +# Install build which is used to get the metadata of package config files. +COPY .generator/requirements.in . +RUN python3.9 -m pip install -r requirements.in + # Copy the CLI script into the container. COPY .generator/cli.py . RUN chmod a+rx ./cli.py diff --git a/.generator/requirements.in b/.generator/requirements.in new file mode 100644 index 000000000000..c795b054e5ad --- /dev/null +++ b/.generator/requirements.in @@ -0,0 +1 @@ +build \ No newline at end of file From b010834f586d93550b1b7a3e2385317b819b0ab1 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 16 Sep 2025 17:33:03 +0000 Subject: [PATCH 6/6] add nit --- .generator/requirements-test.in | 2 +- .generator/requirements.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.generator/requirements-test.in b/.generator/requirements-test.in index b769b9b6d7b4..80bf22b7b8d6 100644 --- a/.generator/requirements-test.in +++ b/.generator/requirements-test.in @@ -17,4 +17,4 @@ pytest-cov pytest-mock gcp-synthtool @ git+https://github.com/googleapis/synthtool@5aa438a342707842d11fbbb302c6277fbf9e4655 starlark-pyo3>=2025.1 -build \ No newline at end of file +build diff --git a/.generator/requirements.in b/.generator/requirements.in index c795b054e5ad..378eac25d311 100644 --- a/.generator/requirements.in +++ b/.generator/requirements.in @@ -1 +1 @@ -build \ No newline at end of file +build