diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 4caf0c45ff..877757e719 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -30,7 +30,7 @@ import os from pathlib import Path from typing import List -import library_generation.utilities as util +import library_generation.utils.utilities as util from library_generation.model.generation_config import GenerationConfig from library_generation.model.gapic_config import GapicConfig from library_generation.model.gapic_inputs import GapicInputs diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index a23cf14653..a5d73ebec4 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -74,7 +74,7 @@ done script_dir=$(dirname "$(readlink -f "$0")") # source utility functions -source "${script_dir}"/utilities.sh +source "${script_dir}"/utils/utilities.sh output_folder="$(get_output_folder)" if [ -z "${gapic_generator_version}" ]; then diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index 8048f810a4..2af636d2fe 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -19,10 +19,8 @@ import click from git import Commit, Repo from library_generation.model.generation_config import from_yaml -from library_generation.utilities import find_versioned_proto_path +from library_generation.utils.proto_path_utils import find_versioned_proto_path from library_generation.utils.commit_message_formatter import format_commit_message -from library_generation.utilities import get_file_paths -from library_generation.utils.commit_message_formatter import wrap_nested_commit from library_generation.utils.commit_message_formatter import wrap_override_commit @@ -87,7 +85,7 @@ def generate_pr_descriptions( baseline_commit: str, ) -> str: config = from_yaml(generation_config_yaml) - paths = get_file_paths(config) + paths = config.get_proto_path_to_library_name() return __get_commit_messages( repo_url=repo_url, latest_commit=config.googleapis_commitish, diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index a4a4a826f2..9af3a08c56 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import library_generation.utilities as util +import library_generation.utils.utilities as util import click import os from library_generation.generate_composed_library import generate_composed_library diff --git a/library_generation/model/config_change.py b/library_generation/model/config_change.py new file mode 100644 index 0000000000..5b1a97d22a --- /dev/null +++ b/library_generation/model/config_change.py @@ -0,0 +1,158 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import shutil +from enum import Enum +from typing import Optional +from git import Commit, Repo +from library_generation.model.generation_config import GenerationConfig +from library_generation.model.library_config import LibraryConfig +from library_generation.utils.utilities import sh_util +from library_generation.utils.proto_path_utils import find_versioned_proto_path + + +class ChangeType(Enum): + GOOGLEAPIS_COMMIT = 1 + REPO_LEVEL_CHANGE = 2 + LIBRARIES_ADDITION = 3 + # As of Mar. 2024, we decide not to produce this type of change because we + # still need to manually remove the libray. + # LIBRARIES_REMOVAL = 4 + LIBRARY_LEVEL_CHANGE = 5 + GAPIC_ADDITION = 6 + # As of Mar. 2024, we decide not to produce this type of change because we + # still need to manually remove the libray. + # GAPIC_REMOVAL = 7 + + +class HashLibrary: + """ + Data class to group a LibraryConfig object and its hash value together. + """ + + def __init__(self, hash_value: int, library: LibraryConfig): + self.hash_value = hash_value + self.library = library + + +class LibraryChange: + def __init__(self, changed_param: str, latest_value: str, library_name: str = ""): + self.changed_param = changed_param + self.latest_value = latest_value + self.library_name = library_name + + +class QualifiedCommit: + def __init__(self, commit: Commit, libraries: set[str]): + self.commit = commit + self.libraries = libraries + + +class ConfigChange: + ALL_LIBRARIES_CHANGED = None + + def __init__( + self, + change_to_libraries: dict[ChangeType, list[LibraryChange]], + baseline_config: GenerationConfig, + latest_config: GenerationConfig, + ): + self.change_to_libraries = change_to_libraries + self.baseline_config = baseline_config + self.latest_config = latest_config + + def get_changed_libraries(self) -> Optional[list[str]]: + """ + Returns a unique, sorted list of library name of changed libraries. + None if there is a repository level change, which means all libraries + in the latest_config will be generated. + :return: library names of change libraries. + """ + if ChangeType.REPO_LEVEL_CHANGE in self.change_to_libraries: + return ConfigChange.ALL_LIBRARIES_CHANGED + library_names = set() + for change_type, library_changes in self.change_to_libraries.items(): + if change_type == ChangeType.GOOGLEAPIS_COMMIT: + library_names.update(self.__get_library_names_from_qualified_commits()) + else: + library_names.update( + [library_change.library_name for library_change in library_changes] + ) + return sorted(list(library_names)) + + def get_qualified_commits( + self, + repo_url: str = "https://github.com/googleapis/googleapis.git", + ) -> list[QualifiedCommit]: + """ + Returns qualified commits from configuration change. + + A qualified commit is a commit that changes at least one file (excluding + BUILD.bazel) within a versioned proto path in the given proto_paths. + :param repo_url: the repository contains the commit history. + :return: QualifiedCommit objects. + """ + tmp_dir = sh_util("get_output_folder") + shutil.rmtree(tmp_dir, ignore_errors=True) + os.mkdir(tmp_dir) + # we only need commit history, thus shadow clone is enough. + repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"]) + commit = repo.commit(self.latest_config.googleapis_commitish) + proto_paths = self.latest_config.get_proto_path_to_library_name() + qualified_commits = [] + while str(commit.hexsha) != self.baseline_config.googleapis_commitish: + qualified_commit = ConfigChange.__create_qualified_commit( + proto_paths=proto_paths, commit=commit + ) + if qualified_commit is not None: + qualified_commits.append(qualified_commit) + commit_parents = commit.parents + if len(commit_parents) == 0: + break + commit = commit_parents[0] + shutil.rmtree(tmp_dir, ignore_errors=True) + return qualified_commits + + def __get_library_names_from_qualified_commits(self) -> list[str]: + qualified_commits = self.get_qualified_commits() + library_names = [] + for qualified_commit in qualified_commits: + library_names.extend(qualified_commit.libraries) + return library_names + + @staticmethod + def __create_qualified_commit( + proto_paths: dict[str, str], commit: Commit + ) -> Optional[QualifiedCommit]: + """ + Returns a qualified commit from the given Commit object; otherwise None. + + :param proto_paths: a mapping from versioned proto_path to library_name + :param commit: a GitHub commit object. + :return: qualified commits. + """ + libraries = set() + for file in commit.stats.files.keys(): + if file.endswith("BUILD.bazel"): + continue + versioned_proto_path = find_versioned_proto_path(file) + if versioned_proto_path in proto_paths: + # Even though a commit usually only changes one + # library, we don't want to miss generating a + # library because the commit may change multiple + # libraries. + libraries.add(proto_paths[versioned_proto_path]) + if len(libraries) == 0: + return None + return QualifiedCommit(commit=commit, libraries=libraries) diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index 22823b903d..b37b24b5cb 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -49,6 +49,18 @@ def __init__( # monorepos have more than one library defined in the config yaml self.is_monorepo = len(libraries) > 1 + def get_proto_path_to_library_name(self) -> dict[str, str]: + """ + Get versioned proto_path to library_name mapping from configuration. + + :return: versioned proto_path to library_name mapping + """ + paths = {} + for library in self.libraries: + for gapic_config in library.gapic_configs: + paths[gapic_config.proto_path] = library.get_library_name() + return paths + def from_yaml(path_to_yaml: str) -> GenerationConfig: """ diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index c6a5b4020d..480b7e5170 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -34,7 +34,7 @@ synthtool_commitish=$6 is_monorepo=$7 configuration_yaml_path=$8 -source "${scripts_root}"/utilities.sh +source "${scripts_root}"/utils/utilities.sh declare -a required_inputs=("postprocessing_target" "versions_file" "owlbot_cli_image_sha" "synthtool_commitish" "is_monorepo") for required_input in "${required_inputs[@]}"; do diff --git a/library_generation/setup.py b/library_generation/setup.py index 477346e711..bd17cc5fcf 100755 --- a/library_generation/setup.py +++ b/library_generation/setup.py @@ -12,7 +12,9 @@ }, package_data={ "library_generation": [ - "*.sh", + "generate_library.sh", + "postprocess_library.sh", + "utils/utilities.sh", "templates/*.j2", "gapic-generator-java-wrapper", "requirements.*", diff --git a/library_generation/test/compare_poms.py b/library_generation/test/compare_poms.py index f58953505d..a1ef443799 100644 --- a/library_generation/test/compare_poms.py +++ b/library_generation/test/compare_poms.py @@ -4,7 +4,7 @@ The only comparison points are: element path (e.g. project/dependencies) and element text There is a special case for `dependency`, where the maven coordinates are prepared as well """ -from library_generation.utilities import eprint +from library_generation.utils.utilities import eprint import xml.etree.ElementTree as et from collections import Counter import sys diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index e9f4954298..ac499435c8 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -5,7 +5,7 @@ set -xeo pipefail # Unit tests against ../utilities.sh script_dir=$(dirname "$(readlink -f "$0")") source "${script_dir}"/test_utilities.sh -source "${script_dir}"/../utilities.sh +source "${script_dir}"/../utils/utilities.sh # Unit tests extract_folder_name_test() { diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 35cbf565ac..9c1d84a33c 100755 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -28,7 +28,7 @@ from library_generation.generate_repo import generate_from_yaml from library_generation.model.generation_config import from_yaml, GenerationConfig from library_generation.test.compare_poms import compare_xml -from library_generation.utilities import ( +from library_generation.utils.utilities import ( sh_util as shell_call, run_process_and_print_output, ) diff --git a/library_generation/test/model/config_change_unit_tests.py b/library_generation/test/model/config_change_unit_tests.py new file mode 100644 index 0000000000..d6309c98c1 --- /dev/null +++ b/library_generation/test/model/config_change_unit_tests.py @@ -0,0 +1,264 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.model.config_change import ChangeType +from library_generation.model.config_change import ConfigChange +from library_generation.model.config_change import LibraryChange +from library_generation.model.gapic_config import GapicConfig +from library_generation.model.generation_config import GenerationConfig +from library_generation.model.library_config import LibraryConfig + + +class ConfigChangeTest(unittest.TestCase): + def test_get_changed_libraries_with_repo_level_change_returns_all_libraries_changed( + self, + ): + config_change = ConfigChange( + change_to_libraries={ + ChangeType.REPO_LEVEL_CHANGE: [], + # add a library level change to verify this type of change has + # no impact on the result. + ChangeType.LIBRARY_LEVEL_CHANGE: [ + LibraryChange( + changed_param="test", + latest_value="test", + library_name="test-library", + ) + ], + }, + baseline_config=ConfigChangeTest.__get_a_gen_config(), + latest_config=ConfigChangeTest.__get_a_gen_config(), + ) + self.assertEqual( + ConfigChange.ALL_LIBRARIES_CHANGED, + config_change.get_changed_libraries(), + ) + + def test_get_changed_libraries_with_library_level_change_returns_list(self): + config_change = ConfigChange( + change_to_libraries={ + ChangeType.LIBRARY_LEVEL_CHANGE: [ + LibraryChange( + changed_param="test", + latest_value="test", + library_name="a-library", + ), + LibraryChange( + changed_param="test", + latest_value="test", + library_name="another-library", + ), + ], + ChangeType.LIBRARIES_ADDITION: [ + LibraryChange( + changed_param="test", + latest_value="test", + library_name="new-library", + ), + ], + ChangeType.GAPIC_ADDITION: [ + LibraryChange( + changed_param="test", + latest_value="test", + library_name="library-with-new-version", + ), + ], + }, + baseline_config=ConfigChangeTest.__get_a_gen_config(), + latest_config=ConfigChangeTest.__get_a_gen_config(), + ) + self.assertEqual( + ["a-library", "another-library", "library-with-new-version", "new-library"], + config_change.get_changed_libraries(), + ) + + def test_get_changed_libraries_with_duplicated_library_name_returns_unique_name( + self, + ): + config_change = ConfigChange( + change_to_libraries={ + ChangeType.LIBRARY_LEVEL_CHANGE: [ + LibraryChange( + changed_param="a-param", + latest_value="new_test", + library_name="a-library", + ), + LibraryChange( + changed_param="another-param", + latest_value="new_value", + library_name="a-library", + ), + ], + }, + baseline_config=ConfigChangeTest.__get_a_gen_config(), + latest_config=ConfigChangeTest.__get_a_gen_config(), + ) + self.assertEqual( + ["a-library"], + config_change.get_changed_libraries(), + ) + + def test_get_changed_libraries_with_mix_changes_returns_list(self): + baseline_commit = "277145d108819fa30fbed3a7cbbb50f91eb6155e" + latest_commit = "8984ddb508dea0e673b724c58338e810b1d8aee3" + config_change = ConfigChange( + change_to_libraries={ + ChangeType.GOOGLEAPIS_COMMIT: [], + ChangeType.LIBRARY_LEVEL_CHANGE: [ + LibraryChange( + changed_param="test", + latest_value="test", + library_name="a-library", + ) + ], + ChangeType.LIBRARIES_ADDITION: [ + LibraryChange( + changed_param="test", + latest_value="test", + library_name="new-library", + ), + ], + }, + baseline_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=baseline_commit + ), + latest_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=latest_commit, + libraries=[ + ConfigChangeTest.__get_a_library_config( + library_name="gke-backup", + gapic_configs=[ + GapicConfig(proto_path="google/cloud/gkebackup/v1") + ], + ), + ], + ), + ) + + self.assertEqual( + ["a-library", "gke-backup", "new-library"], + sorted(config_change.get_changed_libraries()), + ) + + def test_get_qualified_commits_success(self): + baseline_commit = "277145d108819fa30fbed3a7cbbb50f91eb6155e" + latest_commit = "8984ddb508dea0e673b724c58338e810b1d8aee3" + gke_backup_commit = "b8691edb3f1d3c1583aa9cd89240eb359eebe9c7" + aiplatform_commit = "b82095baef02e525bee7bb1c48911c33b66acdf0" + network_management_commit = "efad09c9f0d46ae0786d810a88024363e06c6ca3" + config_change = ConfigChange( + change_to_libraries={}, + baseline_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=baseline_commit + ), + latest_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=latest_commit, + libraries=[ + ConfigChangeTest.__get_a_library_config( + library_name="gke-backup", + gapic_configs=[ + GapicConfig(proto_path="google/cloud/gkebackup/v1") + ], + ), + ConfigChangeTest.__get_a_library_config( + library_name="aiplatform", + gapic_configs=[ + GapicConfig(proto_path="google/cloud/aiplatform/v1beta1") + ], + ), + ConfigChangeTest.__get_a_library_config( + library_name="network-management", + gapic_configs=[ + GapicConfig(proto_path="google/cloud/networkmanagement/v1"), + GapicConfig( + proto_path="google/cloud/networkmanagement/v1beta1" + ), + ], + ), + ], + ), + ) + qualified_commits = config_change.get_qualified_commits() + self.assertEqual(3, len(qualified_commits)) + self.assertEqual({"gke-backup"}, qualified_commits[0].libraries) + self.assertEqual( + gke_backup_commit, + qualified_commits[0].commit.hexsha, + ) + self.assertEqual({"aiplatform"}, qualified_commits[1].libraries) + self.assertEqual( + aiplatform_commit, + qualified_commits[1].commit.hexsha, + ) + self.assertEqual({"network-management"}, qualified_commits[2].libraries) + self.assertEqual( + network_management_commit, + qualified_commits[2].commit.hexsha, + ) + + def test_get_qualified_commits_build_only_commit_returns_empty_list(self): + baseline_commit = "bdda0174f68a738518ec311e05e6fd9bbe19cd78" + latest_commit = "c9a5050ef225b0011603e1109cf53ab1de0a8e53" + config_change = ConfigChange( + change_to_libraries={}, + baseline_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=baseline_commit + ), + latest_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=latest_commit, + libraries=[ + ConfigChangeTest.__get_a_library_config( + library_name="chat", + gapic_configs=[GapicConfig(proto_path="google/chat/v1")], + ) + ], + ), + ) + # one commit between latest_commit and baseline_commit which only + # changed BUILD.bazel. + self.assertTrue(len(config_change.get_qualified_commits()) == 0) + + @staticmethod + def __get_a_gen_config( + googleapis_commitish="", libraries: list[LibraryConfig] = None + ) -> GenerationConfig: + if libraries is None: + libraries = [] + return GenerationConfig( + gapic_generator_version="", + googleapis_commitish=googleapis_commitish, + owlbot_cli_image="", + synthtool_commitish="", + template_excludes=[], + path_to_yaml="", + grpc_version="", + protobuf_version="", + libraries=libraries, + ) + + @staticmethod + def __get_a_library_config( + library_name: str, gapic_configs: list[GapicConfig] = None + ) -> LibraryConfig: + if gapic_configs is None: + gapic_configs = [] + return LibraryConfig( + api_shortname="existing_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=gapic_configs, + library_name=library_name, + ) diff --git a/library_generation/test/model/generation_config_unit_test.py b/library_generation/test/model/generation_config_unit_test.py new file mode 100644 index 0000000000..b056c70979 --- /dev/null +++ b/library_generation/test/model/generation_config_unit_test.py @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +from pathlib import Path + +from library_generation.model.generation_config import from_yaml + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") +test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() + + +class GenerationConfigTest(unittest.TestCase): + def test_get_proto_path_to_library_name_success(self): + paths = from_yaml( + f"{test_config_dir}/generation_config.yaml" + ).get_proto_path_to_library_name() + self.assertEqual( + { + "google/cloud/asset/v1": "asset", + "google/cloud/asset/v1p1beta1": "asset", + "google/cloud/asset/v1p2beta1": "asset", + "google/cloud/asset/v1p5beta1": "asset", + "google/cloud/asset/v1p7beta1": "asset", + }, + paths, + ) diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index 3d114f977f..706570ccc1 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -22,7 +22,7 @@ import contextlib from pathlib import Path from parameterized import parameterized -from library_generation import utilities as util +from library_generation.utils import utilities as util from library_generation.model.gapic_config import GapicConfig from library_generation.model.generation_config import GenerationConfig from library_generation.model.gapic_inputs import parse as parse_build_file @@ -30,8 +30,6 @@ from library_generation.model.library_config import LibraryConfig from library_generation.test.test_utils import FileComparator from library_generation.test.test_utils import cleanup -from library_generation.utilities import find_versioned_proto_path -from library_generation.utilities import get_file_paths script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, "resources") @@ -217,36 +215,6 @@ def test_from_yaml_succeeds(self): self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path) self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path) - def test_get_file_paths_from_yaml_success(self): - paths = get_file_paths(from_yaml(f"{test_config_dir}/generation_config.yaml")) - self.assertEqual( - { - "google/cloud/asset/v1": "asset", - "google/cloud/asset/v1p1beta1": "asset", - "google/cloud/asset/v1p2beta1": "asset", - "google/cloud/asset/v1p5beta1": "asset", - "google/cloud/asset/v1p7beta1": "asset", - }, - paths, - ) - - @parameterized.expand( - [ - ( - "google/cloud/aiplatform/v1/schema/predict/params/image_classification.proto", - "google/cloud/aiplatform/v1", - ), - ( - "google/cloud/asset/v1p2beta1/assets.proto", - "google/cloud/asset/v1p2beta1", - ), - ("google/type/color.proto", "google/type/color.proto"), - ] - ) - def test_find_versioned_proto_path(self, file_path, expected): - proto_path = find_versioned_proto_path(file_path) - self.assertEqual(expected, proto_path) - @parameterized.expand( [ ("BUILD_no_additional_protos.bazel", " "), @@ -379,18 +347,6 @@ def test_gapic_inputs_parse_no_service_yaml_returns_empty_string(self): ) self.assertEqual("", parsed.service_yaml) - def test_remove_version_from_returns_non_versioned_path(self): - proto_path = "google/cloud/aiplatform/v1" - self.assertEqual( - "google/cloud/aiplatform", util.remove_version_from(proto_path) - ) - - def test_remove_version_from_returns_self(self): - proto_path = "google/cloud/aiplatform" - self.assertEqual( - "google/cloud/aiplatform", util.remove_version_from(proto_path) - ) - def test_generate_prerequisite_files_non_monorepo_success(self): library_path = self.__setup_prerequisite_files( num_libraries=1, library_type="GAPIC_COMBO" diff --git a/library_generation/test/utils/generation_config_comparator_unit_tests.py b/library_generation/test/utils/generation_config_comparator_unit_tests.py index c2c025c579..e526d6856b 100644 --- a/library_generation/test/utils/generation_config_comparator_unit_tests.py +++ b/library_generation/test/utils/generation_config_comparator_unit_tests.py @@ -64,7 +64,7 @@ def test_compare_config_not_change(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result) == 0) + self.assertTrue(len(result.change_to_libraries) == 0) def test_compare_config_googleapis_update(self): self.baseline_config.googleapis_commitish = ( @@ -77,7 +77,7 @@ def test_compare_config_googleapis_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertEqual({ChangeType.GOOGLEAPIS_COMMIT: []}, result) + self.assertEqual({ChangeType.GOOGLEAPIS_COMMIT: []}, result.change_to_libraries) def test_compare_config_generator_update(self): self.baseline_config.gapic_generator_version = "1.2.3" @@ -86,8 +86,10 @@ def test_compare_config_generator_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("gapic_generator_version", config_change.changed_param) self.assertEqual("1.2.4", config_change.latest_value) @@ -98,8 +100,10 @@ def test_compare_config_owlbot_cli_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("owlbot_cli_image", config_change.changed_param) self.assertEqual("image_version_456", config_change.latest_value) @@ -110,8 +114,10 @@ def test_compare_config_synthtool_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("synthtool_commitish", config_change.changed_param) self.assertEqual("commit456", config_change.latest_value) @@ -122,8 +128,10 @@ def test_compare_protobuf_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("protobuf_version", config_change.changed_param) self.assertEqual("3.27.0", config_change.latest_value) @@ -134,8 +142,10 @@ def test_compare_config_grpc_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("grpc_version", config_change.changed_param) self.assertEqual("1.61.0", config_change.latest_value) @@ -151,8 +161,10 @@ def test_compare_config_template_excludes_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.REPO_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("template_excludes", config_change.changed_param) self.assertEqual( [ @@ -178,8 +190,10 @@ def test_compare_config_library_addition(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARIES_ADDITION]) == 1) - config_change = result[ChangeType.LIBRARIES_ADDITION][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] self.assertEqual("new_library", config_change.library_name) def test_compare_config_api_shortname_update_without_library_name(self): @@ -188,8 +202,10 @@ def test_compare_config_api_shortname_update_without_library_name(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARIES_ADDITION]) == 1) - config_change = result[ChangeType.LIBRARIES_ADDITION][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] self.assertEqual("new_api_shortname", config_change.library_name) def test_compare_config_api_shortname_update_with_library_name_raise_error(self): @@ -210,8 +226,10 @@ def test_compare_config_library_name_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARIES_ADDITION]) == 1) - config_change = result[ChangeType.LIBRARIES_ADDITION][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] self.assertEqual("new_library_name", config_change.library_name) def test_compare_config_api_description_update(self): @@ -220,8 +238,10 @@ def test_compare_config_api_description_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_description", config_change.changed_param) self.assertEqual("updated description", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -232,8 +252,10 @@ def test_compare_config_name_pretty_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("name_pretty", config_change.changed_param) self.assertEqual("new name", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -244,8 +266,10 @@ def test_compare_config_product_docs_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("product_documentation", config_change.changed_param) self.assertEqual("new docs", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -256,8 +280,10 @@ def test_compare_config_library_type_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("library_type", config_change.changed_param) self.assertEqual("GAPIC_COMBO", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -268,8 +294,10 @@ def test_compare_config_release_level_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("release_level", config_change.changed_param) self.assertEqual("STABLE", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -280,8 +308,10 @@ def test_compare_config_api_id_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_id", config_change.changed_param) self.assertEqual("new_id", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -292,8 +322,10 @@ def test_compare_config_api_reference_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_reference", config_change.changed_param) self.assertEqual("new api_reference", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -304,8 +336,10 @@ def test_compare_config_code_owner_team_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("codeowner_team", config_change.changed_param) self.assertEqual("new team", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -316,8 +350,10 @@ def test_compare_config_excluded_deps_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("excluded_dependencies", config_change.changed_param) self.assertEqual("group:artifact", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -328,8 +364,10 @@ def test_compare_config_excluded_poms_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("excluded_poms", config_change.changed_param) self.assertEqual("pom.xml", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -340,8 +378,10 @@ def test_compare_config_client_docs_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("client_documentation", config_change.changed_param) self.assertEqual("new client docs", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -352,8 +392,10 @@ def test_compare_config_distribution_name_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("distribution_name", config_change.changed_param) self.assertEqual("new_group:new_artifact", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -364,8 +406,10 @@ def test_compare_config_group_id_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("group_id", config_change.changed_param) self.assertEqual("new_group", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -376,8 +420,10 @@ def test_compare_config_issue_tracker_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("issue_tracker", config_change.changed_param) self.assertEqual("new issue tracker", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -388,8 +434,10 @@ def test_compare_config_rest_docs_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("rest_documentation", config_change.changed_param) self.assertEqual("new rest docs", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -400,8 +448,10 @@ def test_compare_config_rpc_docs_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("rpc_documentation", config_change.changed_param) self.assertEqual("new rpc docs", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -412,8 +462,10 @@ def test_compare_config_cloud_api_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("cloud_api", config_change.changed_param) self.assertEqual(False, config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -424,8 +476,10 @@ def test_compare_config_requires_billing_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("requires_billing", config_change.changed_param) self.assertEqual(False, config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -436,8 +490,10 @@ def test_compare_config_extra_versioned_mod_update(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1) - config_change = result[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("extra_versioned_modules", config_change.changed_param) self.assertEqual("extra module", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) @@ -450,8 +506,8 @@ def test_compare_config_version_addition(self): baseline_config=self.baseline_config, latest_config=self.latest_config, ) - self.assertTrue(len(result[ChangeType.GAPIC_ADDITION]) == 1) - config_change = result[ChangeType.GAPIC_ADDITION][0] + self.assertTrue(len(result.change_to_libraries[ChangeType.GAPIC_ADDITION]) == 1) + config_change = result.change_to_libraries[ChangeType.GAPIC_ADDITION][0] self.assertEqual("", config_change.changed_param) self.assertEqual("google/new/library/v1", config_change.latest_value) self.assertEqual("existing_library", config_change.library_name) diff --git a/library_generation/test/utils/proto_path_utils_unit_tests.py b/library_generation/test/utils/proto_path_utils_unit_tests.py new file mode 100644 index 0000000000..2e23c2b403 --- /dev/null +++ b/library_generation/test/utils/proto_path_utils_unit_tests.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +from pathlib import Path +from library_generation.utils.proto_path_utils import ( + find_versioned_proto_path, + remove_version_from, +) + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") +test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() + + +class ProtoPathsUtilsTest(unittest.TestCase): + def test_find_versioned_proto_path_nested_version_success(self): + proto_path = "google/cloud/aiplatform/v1/schema/predict/params/image_classification.proto" + expected = "google/cloud/aiplatform/v1" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_find_versioned_proto_path_success(self): + proto_path = "google/cloud/asset/v1p2beta1/assets.proto" + expected = "google/cloud/asset/v1p2beta1" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_find_versioned_proto_without_version_return_itself(self): + proto_path = "google/type/color.proto" + expected = "google/type/color.proto" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_remove_version_from_returns_non_versioned_path(self): + proto_path = "google/cloud/aiplatform/v1" + self.assertEqual("google/cloud/aiplatform", remove_version_from(proto_path)) + + def test_remove_version_from_returns_self(self): + proto_path = "google/cloud/aiplatform" + self.assertEqual("google/cloud/aiplatform", remove_version_from(proto_path)) diff --git a/library_generation/utils/__init__.py b/library_generation/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/utils/generation_config_comparator.py b/library_generation/utils/generation_config_comparator.py index 4fe8358230..79a5f3c1aa 100644 --- a/library_generation/utils/generation_config_comparator.py +++ b/library_generation/utils/generation_config_comparator.py @@ -12,51 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. from collections import defaultdict -from enum import Enum from typing import Any - from typing import Dict from typing import List - +from library_generation.model.config_change import ChangeType +from library_generation.model.config_change import ConfigChange +from library_generation.model.config_change import LibraryChange +from library_generation.model.config_change import HashLibrary from library_generation.model.gapic_config import GapicConfig from library_generation.model.generation_config import GenerationConfig from library_generation.model.library_config import LibraryConfig -class ChangeType(Enum): - GOOGLEAPIS_COMMIT = 1 - REPO_LEVEL_CHANGE = 2 - LIBRARIES_ADDITION = 3 - # As of Mar. 2024, we decide not to produce this type of change because we - # still need to manually remove the libray. - # LIBRARIES_REMOVAL = 4 - LIBRARY_LEVEL_CHANGE = 5 - GAPIC_ADDITION = 6 - # As of Mar. 2024, we decide not to produce this type of change because we - # still need to manually remove the libray. - # GAPIC_REMOVAL = 7 - - -class HashLibrary: - """ - Data class to group a LibraryConfig object and its hash value together. - """ - - def __init__(self, hash_value: int, library: LibraryConfig): - self.hash_value = hash_value - self.library = library - - -class ConfigChange: - def __init__(self, changed_param: str, latest_value: str, library_name: str = ""): - self.changed_param = changed_param - self.latest_value = latest_value - self.library_name = library_name - - def compare_config( baseline_config: GenerationConfig, latest_config: GenerationConfig -) -> Dict[ChangeType, list[ConfigChange]]: +) -> ConfigChange: """ Compare two GenerationConfig object and output a mapping from ConfigChange to a list of ConfigChange objects. @@ -65,9 +35,9 @@ def compare_config( :param baseline_config: the baseline GenerationConfig object :param latest_config: the latest GenerationConfig object - :return: a mapping from ConfigChange to a list of ConfigChange objects. + :return: a ConfigChange objects. """ - diff = defaultdict(list[ConfigChange]) + diff = defaultdict(list[LibraryChange]) baseline_params = __convert_params_to_sorted_list(baseline_config) latest_params = __convert_params_to_sorted_list(latest_config) for baseline_param, latest_param in zip(baseline_params, latest_params): @@ -76,7 +46,7 @@ def compare_config( if baseline_param[0] == "googleapis_commitish": diff[ChangeType.GOOGLEAPIS_COMMIT] = [] else: - config_change = ConfigChange( + config_change = LibraryChange( changed_param=latest_param[0], latest_value=latest_param[1], ) @@ -87,11 +57,15 @@ def compare_config( baseline_library_configs=baseline_config.libraries, latest_library_configs=latest_config.libraries, ) - return diff + return ConfigChange( + change_to_libraries=diff, + baseline_config=baseline_config, + latest_config=latest_config, + ) def __compare_libraries( - diff: Dict[ChangeType, list[ConfigChange]], + diff: Dict[ChangeType, list[LibraryChange]], baseline_library_configs: List[LibraryConfig], latest_library_configs: List[LibraryConfig], ) -> None: @@ -132,7 +106,7 @@ def __compare_libraries( # a library is added to latest_libraries if the library_name # is not in baseline_libraries. if library_name not in baseline_libraries: - config_change = ConfigChange( + config_change = LibraryChange( changed_param="", latest_value="", library_name=library_name ) diff[ChangeType.LIBRARIES_ADDITION].append(config_change) @@ -163,7 +137,7 @@ def __convert_to_hashed_library_dict( def __compare_changed_libraries( - diff: Dict[ChangeType, list[ConfigChange]], + diff: Dict[ChangeType, list[LibraryChange]], baseline_libraries: Dict[str, HashLibrary], latest_libraries: Dict[str, HashLibrary], changed_libraries: List[str], @@ -193,7 +167,7 @@ def __compare_changed_libraries( f"{library_name}: api_shortname must not change when library_name remains the same." ) else: - config_change = ConfigChange( + config_change = LibraryChange( changed_param=latest_param[0], latest_value=latest_param[1], library_name=library_name, @@ -212,7 +186,7 @@ def __compare_changed_libraries( def __compare_gapic_configs( - diff: Dict[ChangeType, list[ConfigChange]], + diff: Dict[ChangeType, list[LibraryChange]], library_name: str, baseline_gapic_configs: List[GapicConfig], latest_gapic_configs: List[GapicConfig], @@ -236,7 +210,7 @@ def __compare_gapic_configs( for proto_path in latest_proto_paths: if proto_path in baseline_proto_paths: continue - config_change = ConfigChange( + config_change = LibraryChange( changed_param="", latest_value=proto_path, library_name=library_name ) diff[ChangeType.GAPIC_ADDITION].append(config_change) diff --git a/library_generation/utils/proto_path_utils.py b/library_generation/utils/proto_path_utils.py new file mode 100644 index 0000000000..d2ae25f602 --- /dev/null +++ b/library_generation/utils/proto_path_utils.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + + +def remove_version_from(proto_path: str) -> str: + """ + Remove the version of a proto_path + :param proto_path: versioned proto_path + :return: the proto_path without version + """ + version_pattern = "^v[1-9]" + index = proto_path.rfind("/") + version = proto_path[index + 1 :] + if re.match(version_pattern, version): + return proto_path[:index] + return proto_path + + +def find_versioned_proto_path(proto_path: str) -> str: + """ + Returns a versioned proto_path from a given proto_path; or proto_path itself + if it doesn't contain a versioned proto_path. + :param proto_path: a proto file path + :return: the versioned proto_path + """ + version_regex = re.compile(r"^v[1-9].*") + directories = proto_path.split("/") + for directory in directories: + result = version_regex.search(directory) + if result: + version = result[0] + idx = proto_path.find(version) + return proto_path[:idx] + version + return proto_path diff --git a/library_generation/utilities.py b/library_generation/utils/utilities.py similarity index 87% rename from library_generation/utilities.py rename to library_generation/utils/utilities.py index fb7e8938d2..8ab1a11b7c 100755 --- a/library_generation/utilities.py +++ b/library_generation/utils/utilities.py @@ -18,12 +18,12 @@ import shutil import re from pathlib import Path -from typing import Dict from library_generation.model.generation_config import GenerationConfig from library_generation.model.library_config import LibraryConfig from typing import List from library_generation.model.repo_config import RepoConfig from library_generation.utils.file_render import render +from library_generation.utils.proto_path_utils import remove_version_from script_dir = os.path.dirname(os.path.realpath(__file__)) @@ -57,7 +57,7 @@ def run_process_and_print_output(arguments: List[str], job_name: str = "Job"): def sh_util(statement: str, **kwargs) -> str: """ - Calls a function defined in library_generation/utilities.sh + Calls a function defined in library_generation/utils/utilities.sh """ if "stdout" not in kwargs: kwargs["stdout"] = subprocess.PIPE @@ -65,7 +65,11 @@ def sh_util(statement: str, **kwargs) -> str: kwargs["stderr"] = subprocess.PIPE output = "" with subprocess.Popen( - ["bash", "-exc", f"source {script_dir}/utilities.sh && {statement}"], + [ + "bash", + "-exc", + f"source {script_dir}/utilities.sh && {statement}", + ], **kwargs, ) as proc: print("command stderr:") @@ -93,20 +97,6 @@ def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -def remove_version_from(proto_path: str) -> str: - """ - Remove the version of a proto_path - :param proto_path: versioned proto_path - :return: the proto_path without version - """ - version_pattern = "^v[1-9]" - index = proto_path.rfind("/") - version = proto_path[index + 1 :] - if re.match(version_pattern, version): - return proto_path[:index] - return proto_path - - def prepare_repo( gen_config: GenerationConfig, library_config: List[LibraryConfig], @@ -314,36 +304,3 @@ def generate_prerequisite_files( should_include_templates=True, template_excludes=config.template_excludes, ) - - -def get_file_paths(config: GenerationConfig) -> Dict[str, str]: - """ - Get versioned proto_path to library_name mapping from configuration file. - - :param config: a GenerationConfig object. - :return: versioned proto_path to library_name mapping - """ - paths = {} - for library in config.libraries: - for gapic_config in library.gapic_configs: - paths[gapic_config.proto_path] = library.get_library_name() - return paths - - -def find_versioned_proto_path(file_path: str) -> str: - """ - Returns a versioned proto_path from a given file_path; or file_path itself - if it doesn't contain a versioned proto_path. - - :param file_path: a proto file path - :return: the versioned proto_path - """ - version_regex = re.compile(r"^v[1-9].*") - directories = file_path.split("/") - for directory in directories: - result = version_regex.search(directory) - if result: - version = result[0] - idx = file_path.find(version) - return file_path[:idx] + version - return file_path diff --git a/library_generation/utilities.sh b/library_generation/utils/utilities.sh similarity index 100% rename from library_generation/utilities.sh rename to library_generation/utils/utilities.sh diff --git a/showcase/scripts/generate_showcase.sh b/showcase/scripts/generate_showcase.sh index 1c1b1f58de..d524aaa77f 100755 --- a/showcase/scripts/generate_showcase.sh +++ b/showcase/scripts/generate_showcase.sh @@ -8,7 +8,7 @@ set -ex readonly SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) lib_gen_scripts_dir="${SCRIPT_DIR}/../../library_generation/" source "${lib_gen_scripts_dir}/test/test_utilities.sh" -source "${lib_gen_scripts_dir}/utilities.sh" +source "${lib_gen_scripts_dir}/utils/utilities.sh" readonly perform_cleanup=$1 cd "${SCRIPT_DIR}"