From 1114f1863bf5ae15ee56631fd824a2269e91746d Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 13 May 2024 16:25:13 -0400 Subject: [PATCH] fix: prepare to generate grafeas (#2761) In this PR: - Add suffix to sample source folder - Add unit tests Context: - The package name of grafeas is `io.grafeas.v1` which is not starts with `com`. When copying samples to destination directory, we need to change the suffix of source directory. - grafeas doesn't have a valid api_id (empty string). Before this change, the `api_id` will be populated as `{library.api_shortname}.googleapis.com`, which is not correct. `api_id` should not be populated to `.repo-metadata.json` if set to empty string. With the library entry added in generation configuration: ``` - api_shortname: containeranalysis name_pretty: Grafeas product_documentation: https://grafeas.io api_description: n/a client_documentation: "https://cloud.google.com/java/docs/reference/grafeas/latest/overview" release_level: stable distribution_name: "io.grafeas:grafeas" api_id: "" codeowner_team: "@googleapis/aap-dpes" library_name: grafeas requires_billing: false GAPICs: - proto_path: grafeas/v1 ``` The generated java-grafeas is in https://github.com/googleapis/google-cloud-java/pull/10820 --- .../src/main/java/example_main.txt | 0 .../src/test/java/example_test.txt | 0 .../src/main/java/example_proto_main.txt | 0 .../src/main/java/com/example_com_sample.txt | 0 .../src/main/java/io/example_io_sample.txt | 0 .../test/utilities_unit_tests.py | 84 ++++++++++++++++--- library_generation/utils/utilities.sh | 13 ++- 7 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt create mode 100644 library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt create mode 100644 library_generation/test/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt create mode 100644 library_generation/test/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt create mode 100644 library_generation/test/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt diff --git a/library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt b/library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt b/library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/test/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt b/library_generation/test/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/test/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt b/library_generation/test/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/test/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt b/library_generation/test/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index 736eba26e4..96c03087fa 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -102,6 +102,65 @@ def test_sh_util_nonexistent_function_fails(self): with self.assertRaises(RuntimeError): result = util.sh_util("nonexistent_function") + def test_mv_src_files_gapic_main_succeeds(self): + previous_dir = os.getcwd() + os.chdir(f"{resources_dir}/test_mv_src/gapic") + os.environ["folder_name"] = "example" + util.sh_util("mv_src_files gapic main destination") + self.assertTrue( + os.path.isfile("destination/gapic-example/src/main/java/example_main.txt") + ) + shutil.rmtree("destination/gapic-example") + os.chdir(previous_dir) + + def test_mv_src_files_gapic_test_succeeds(self): + previous_dir = os.getcwd() + os.chdir(f"{resources_dir}/test_mv_src/gapic") + os.environ["folder_name"] = "example" + util.sh_util("mv_src_files gapic test destination") + self.assertTrue( + os.path.isfile("destination/gapic-example/src/test/java/example_test.txt") + ) + shutil.rmtree("destination/gapic-example") + os.chdir(previous_dir) + + def test_mv_src_files_proto_main_succeeds(self): + previous_dir = os.getcwd() + os.chdir(f"{resources_dir}/test_mv_src/proto") + os.environ["folder_name"] = "example" + util.sh_util("mv_src_files proto main destination") + self.assertTrue( + os.path.isfile( + "destination/proto-example/src/main/java/example_proto_main.txt" + ) + ) + shutil.rmtree("destination/proto-example") + os.chdir(previous_dir) + + def test_mv_src_files_sample_suffix_io_succeeds(self): + previous_dir = os.getcwd() + os.chdir(f"{resources_dir}/test_mv_src/samples") + util.sh_util("mv_src_files samples main destination_io") + self.assertTrue( + os.path.isfile( + "destination_io/samples/snippets/generated/io/example_io_sample.txt" + ) + ) + shutil.rmtree("destination_io/samples") + os.chdir(previous_dir) + + def test_mv_src_files_sample_suffix_com_succeeds(self): + previous_dir = os.getcwd() + os.chdir(f"{resources_dir}/test_mv_src/samples") + util.sh_util("mv_src_files samples main destination_com") + self.assertTrue( + os.path.isfile( + "destination_com/samples/snippets/generated/com/example_com_sample.txt" + ) + ) + shutil.rmtree("destination_com/samples") + os.chdir(previous_dir) + def test_eprint_valid_input_succeeds(self): test_input = "This is some test input" # create a stdio capture object @@ -247,7 +306,7 @@ def test_gapic_inputs_parse_no_service_yaml_returns_empty_string(self): def test_generate_prerequisite_files_non_monorepo_success(self): library_path = self.__setup_prerequisite_files( - num_libraries=1, library_type="GAPIC_COMBO" + combination=1, library_type="GAPIC_COMBO" ) file_comparator.compare_files( @@ -266,7 +325,7 @@ def test_generate_prerequisite_files_non_monorepo_success(self): self.__remove_prerequisite_files(path=library_path, is_monorepo=False) def test_generate_prerequisite_files_monorepo_success(self): - library_path = self.__setup_prerequisite_files(num_libraries=2) + library_path = self.__setup_prerequisite_files(combination=2) file_comparator.compare_files( f"{library_path}/.repo-metadata.json", @@ -317,7 +376,10 @@ def test_prepare_repo_split_repo_success(self): shutil.rmtree(repo_config.output_folder) def __setup_prerequisite_files( - self, num_libraries: int, library_type: str = "GAPIC_AUTO" + self, + combination: int, + library_type: str = "GAPIC_AUTO", + library: LibraryConfig = library_1, ) -> str: library_path = f"{resources_dir}/goldens" files = [ @@ -326,12 +388,12 @@ def __setup_prerequisite_files( f"{library_path}/owlbot.py", ] cleanup(files) - config = self.__get_a_gen_config(num_libraries, library_type=library_type) + config = self.__get_a_gen_config(combination, library_type=library_type) proto_path = "google/cloud/baremetalsolution/v2" transport = "grpc" util.generate_prerequisite_files( config=config, - library=library_1, + library=library, proto_path=proto_path, transport=transport, library_path=library_path, @@ -340,20 +402,20 @@ def __setup_prerequisite_files( @staticmethod def __get_a_gen_config( - num_libraries: int, library_type: str = "GAPIC_AUTO" + combination: int, library_type: str = "GAPIC_AUTO" ) -> GenerationConfig: """ Returns an object of GenerationConfig with one to three of LibraryConfig objects. Other attributes are set to empty str. - :param num_libraries: the number of LibraryConfig objects associated with + :param combination: combination of LibraryConfig objects associated with the GenerationConfig. Only support 1, 2 or 3. :return: an object of GenerationConfig """ - if num_libraries == 2: + if combination == 2: libraries = [library_1, library_2] - elif num_libraries == 3: - libraries = [library_1, library_2, library_3] + elif combination == 3: + libraries = [library_with_empty_api_id, library_2] else: libraries = [library_1] @@ -361,7 +423,7 @@ def __get_a_gen_config( # library_type) for library in libraries: library.library_type = library_type - if num_libraries == 1: + if combination == 1: # treat this as a HW library case to generate a real-life # repo-metadata library.extra_versioned_modules = "test-module" diff --git a/library_generation/utils/utilities.sh b/library_generation/utils/utilities.sh index 16ad766ce5..237d3c23a9 100755 --- a/library_generation/utils/utilities.sh +++ b/library_generation/utils/utilities.sh @@ -32,8 +32,10 @@ mv_src_files() { local type=$2 # one of main, test local destination_path=$3 if [ "${category}" == "samples" ]; then - src_suffix="samples/snippets/generated/src/main/java/com" + src_suffix="samples/snippets/generated/src/main/java" folder_suffix="samples/snippets/generated" + mkdir -p "${destination_path}/${folder_suffix}" + cp -r "${destination_path}/java_gapic_srcjar/${src_suffix}"/* "${destination_path}/${folder_suffix}" elif [ "${category}" == "proto" ]; then src_suffix="${category}/src/${type}/java" folder_suffix="${category}-${folder_name}/src/${type}" @@ -41,11 +43,14 @@ mv_src_files() { src_suffix="src/${type}" folder_suffix="${category}-${folder_name}/src" fi + + if [ "${category}" == "samples" ]; then + return + fi + mkdir -p "${destination_path}/${folder_suffix}" cp -r "${destination_path}/java_gapic_srcjar/${src_suffix}" "${destination_path}/${folder_suffix}" - if [ "${category}" != "samples" ]; then - rm -r -f "${destination_path}/${folder_suffix}/java/META-INF" - fi + rm -r -f "${destination_path}/${folder_suffix}/java/META-INF" } # unzip jar file