feat: generate proto-only repository

java-common-protos/owlbot.py

@@ -44,6 +44,7 @@
     "SECURITY.md",
     "java.header",
     "license-checks.xml",
+    "README.md",
     "renovate.json",
     ".gitignore"
 ])

library_generation/generate_composed_library.py

@@ -47,10 +47,12 @@ def generate_composed_library(
     library: LibraryConfig,
     output_folder: str,
     versions_file: str,
+    gapic_repo: bool,
 ) -> None:
     """
     Generate libraries composed of more than one service or service version
 
+    :param has_proto_only_libraries:
     :param config_path: Path to generation configuration.
     :param config: a GenerationConfig object representing a parsed configuration
     yaml
@@ -59,6 +61,8 @@ def generate_composed_library(
     for convenience and to prevent all libraries to be processed
     :param output_folder: the folder to where tools go
     :param versions_file: the file containing version of libraries
+    :param gapic_repo: whether the library is generated into a gapic
+    repository or not.
     :return None
     """
     util.pull_api_definition(
@@ -81,6 +85,7 @@ def generate_composed_library(
             proto_path=util.remove_version_from(gapic.proto_path),
             transport=gapic_inputs.transport,
             library_path=library_path,
+            gapic_repo=gapic_repo,
         )
         temp_destination_path = f"java-{gapic.proto_path.replace('/','-')}"
         effective_arguments = __construct_effective_arg(

library_generation/generate_repo.py

@@ -18,6 +18,8 @@
 from library_generation.model.library_config import LibraryConfig
 from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing
 
+PROTO_ONLY_LIBRARIES = ["common-protos"]
+
 
 def generate_from_yaml(
     config_path: str,
@@ -48,18 +50,17 @@ def generate_from_yaml(
 
     for library_path, library in repo_config.libraries.items():
         print(f"generating library {library.get_library_name()}")
-
         generate_composed_library(
             config_path=config_path,
             config=config,
             library_path=library_path,
             library=library,
             output_folder=repo_config.output_folder,
             versions_file=repo_config.versions_file,
+            gapic_repo=not config.contains_common_protos(),
         )
 
-    # we skip monorepo_postprocessing if not in a monorepo
-    if not config.is_monorepo():
+    if not config.is_monorepo() or config.contains_common_protos():
         return
 
     monorepo_postprocessing(

library_generation/model/gapic_inputs.py

@@ -56,7 +56,7 @@ def __init__(
         self,
         proto_only="true",
         additional_protos="google/cloud/common_resources.proto",
-        transport="",
+        transport="grpc",
         rest_numeric_enum="",
         gapic_yaml="",
         service_config="",

library_generation/model/generation_config.py

@@ -20,6 +20,7 @@
 REPO_LEVEL_PARAMETER = "Repo level parameter"
 LIBRARY_LEVEL_PARAMETER = "Library level parameter"
 GAPIC_LEVEL_PARAMETER = "GAPIC level parameter"
+COMMON_PROTOS = "common-protos"
 
 
 class GenerationConfig:
@@ -31,9 +32,9 @@ def __init__(
         self,
         gapic_generator_version: str,
         googleapis_commitish: str,
-        libraries_bom_version: str,
         template_excludes: list[str],
         libraries: list[LibraryConfig],
+        libraries_bom_version: Optional[str] = None,
         grpc_version: Optional[str] = None,
         protoc_version: Optional[str] = None,
     ):
@@ -44,6 +45,7 @@ def __init__(
         self.libraries = libraries
         self.grpc_version = grpc_version
         self.protoc_version = protoc_version
+        self.__contains_common_protos = self.__set_contains_common_protos()
         self.__validate()
 
     def get_proto_path_to_library_name(self) -> dict[str, str]:
@@ -61,6 +63,15 @@ def get_proto_path_to_library_name(self) -> dict[str, str]:
     def is_monorepo(self) -> bool:
         return len(self.libraries) > 1
 
+    def contains_common_protos(self) -> bool:
+        return self.__contains_common_protos
+
+    def __set_contains_common_protos(self) -> bool:
+        for library in self.libraries:
+            if library.get_library_name() == COMMON_PROTOS:
+                return True
+        return False
+
     def __validate(self) -> None:
         seen_library_names = dict()
         for library in self.libraries:
@@ -133,15 +144,13 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
         gapic_generator_version=__required(
             config, "gapic_generator_version", REPO_LEVEL_PARAMETER
         ),
-        grpc_version=__optional(config, "grpc_version", None),
-        protoc_version=__optional(config, "protoc_version", None),
         googleapis_commitish=__required(
             config, "googleapis_commitish", REPO_LEVEL_PARAMETER
         ),
-        libraries_bom_version=__required(
-            config, "libraries_bom_version", REPO_LEVEL_PARAMETER
-        ),
         template_excludes=__required(config, "template_excludes", REPO_LEVEL_PARAMETER),
+        grpc_version=__optional(config, "grpc_version", None),
+        protoc_version=__optional(config, "protoc_version", None),
+        libraries_bom_version=__optional(config, "libraries_bom_version", None),
         libraries=parsed_libraries,
     )
 

library_generation/test/cli/entry_point_unit_tests.py

@@ -22,6 +22,7 @@
 
 class EntryPointTest(unittest.TestCase):
     def test_entry_point_without_config_raise_file_exception(self):
+        os.chdir(script_dir)
         runner = CliRunner()
         # noinspection PyTypeChecker
         result = runner.invoke(generate, ["--repository-path=."])

library_generation/test/model/gapic_inputs_unit_tests.py

@@ -0,0 +1,132 @@
+import os
+import unittest
+from pathlib import Path
+
+from parameterized import parameterized
+from library_generation.model.gapic_inputs import parse
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+resources_dir = os.path.join(script_dir, "..", "resources")
+build_file = Path(os.path.join(resources_dir, "misc")).resolve()
+
+
+class UtilitiesTest(unittest.TestCase):
+    @parameterized.expand(
+        [
+            ("BUILD_no_additional_protos.bazel", " "),
+            ("BUILD_common_resources.bazel", "  google/cloud/common_resources.proto"),
+            ("BUILD_comment_common_resources.bazel", " "),
+            ("BUILD_locations.bazel", "  google/cloud/location/locations.proto"),
+            ("BUILD_comment_locations.bazel", " "),
+            ("BUILD_iam_policy.bazel", "  google/iam/v1/iam_policy.proto"),
+            ("BUILD_comment_iam_policy.bazel", " "),
+            (
+                "BUILD_iam_locations.bazel",
+                "  google/cloud/location/locations.proto google/iam/v1/iam_policy.proto",
+            ),
+        ]
+    )
+    def test_gapic_inputs_parse_additional_protos(self, build_name, expected):
+        parsed = parse(build_file, "", build_name)
+        self.assertEqual(
+            expected,
+            parsed.additional_protos,
+        )
+
+    def test_gapic_inputs_parse_grpc_only_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_grpc.bazel")
+        self.assertEqual("grpc", parsed.transport)
+
+    def test_gapic_inputs_parse_grpc_rest_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_grpc_rest.bazel")
+        self.assertEqual("grpc+rest", parsed.transport)
+
+    def test_gapic_inputs_parse_rest_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_rest.bazel")
+        self.assertEqual("rest", parsed.transport)
+
+    def test_gapic_inputs_parse_empty_include_samples_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel")
+        self.assertEqual("false", parsed.include_samples)
+
+    def test_gapic_inputs_parse_include_samples_false_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_include_samples_false.bazel")
+        self.assertEqual("false", parsed.include_samples)
+
+    def test_gapic_inputs_parse_include_samples_true_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_include_samples_true.bazel")
+        self.assertEqual("true", parsed.include_samples)
+
+    def test_gapic_inputs_parse_empty_rest_numeric_enums_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_rest_numeric_enums_empty.bazel")
+        self.assertEqual("false", parsed.rest_numeric_enum)
+
+    def test_gapic_inputs_parse_rest_numeric_enums_false_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_rest_numeric_enums_false.bazel")
+        self.assertEqual("false", parsed.rest_numeric_enum)
+
+    def test_gapic_inputs_parse_rest_numeric_enums_true_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_rest_numeric_enums_true.bazel")
+        self.assertEqual("true", parsed.rest_numeric_enum)
+
+    def test_gapic_inputs_parse_no_gapic_library_returns_proto_only_true(self):
+        # include_samples_empty only has a gradle assembly rule
+        parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel")
+        self.assertEqual("true", parsed.proto_only)
+
+    def test_gapic_inputs_parse_with_gapic_library_returns_proto_only_false(self):
+        # rest.bazel has a java_gapic_library rule
+        parsed = parse(build_file, "", "BUILD_rest.bazel")
+        self.assertEqual("false", parsed.proto_only)
+
+    def test_gapic_inputs_parse_gapic_yaml_succeeds(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_gapic_yaml.bazel")
+        self.assertEqual("test/versioned/path/test_gapic_yaml.yaml", parsed.gapic_yaml)
+
+    def test_gapic_inputs_parse_no_gapic_yaml_returns_empty_string(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_no_gapic_yaml.bazel")
+        self.assertEqual("", parsed.gapic_yaml)
+
+    def test_gapic_inputs_parse_service_config_succeeds(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_service_config.bazel")
+        self.assertEqual(
+            "test/versioned/path/test_service_config.json", parsed.service_config
+        )
+
+    def test_gapic_inputs_parse_service_yaml_relative_target(self):
+        parsed = parse(
+            build_file,
+            "google/cloud/compute/v1",
+            "BUILD_service_config_relative_target.bazel",
+        )
+        self.assertEqual(
+            "google/cloud/compute/v1/compute_grpc_service_config.json",
+            parsed.service_config,
+        )
+
+    def test_gapic_inputs_parse_no_service_config_returns_empty_string(self):
+        parsed = parse(
+            build_file, "test/versioned/path", "BUILD_no_service_config.bazel"
+        )
+        self.assertEqual("", parsed.service_config)
+
+    def test_gapic_inputs_parse_service_yaml_succeeds(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_service_yaml.bazel")
+        self.assertEqual(
+            "test/versioned/path/test_service_yaml.yaml", parsed.service_yaml
+        )
+
+    def test_gapic_inputs_parse_service_yaml_absolute_target(self):
+        parsed = parse(build_file, "", "BUILD_service_yaml_absolute_target.bazel")
+        self.assertEqual(
+            "google/cloud/videointelligence/videointelligence_v1p3beta1.yaml",
+            parsed.service_yaml,
+        )
+
+    def test_gapic_inputs_parse_no_service_yaml_returns_empty_string(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_no_service_yaml.bazel")
+        self.assertEqual("", parsed.service_yaml)
+
+    def test_gapic_inputs_parse_proto_only_returns_grpc(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_proto_only.bazel")
+        self.assertEqual("grpc", parsed.transport)

library_generation/test/model/generation_config_unit_test.py → library_generation/test/model/generation_config_unit_tests.py

@@ -35,6 +35,13 @@
     product_documentation="",
     gapic_configs=[],
 )
+common_protos_library = LibraryConfig(
+    api_shortname="common-protos",
+    api_description="",
+    name_pretty="",
+    product_documentation="",
+    gapic_configs=[],
+)
 
 
 class GenerationConfigTest(unittest.TestCase):
@@ -123,6 +130,26 @@ def test_is_monorepo_with_two_libraries_returns_true(self):
         )
         self.assertTrue(config.is_monorepo())
 
+    def test_contains_common_protos_with_common_protos_returns_true(self):
+        config = GenerationConfig(
+            gapic_generator_version="",
+            googleapis_commitish="",
+            libraries_bom_version="",
+            template_excludes=[],
+            libraries=[library_1, library_2, common_protos_library],
+        )
+        self.assertTrue(config.contains_common_protos())
+
+    def test_contains_common_protos_without_common_protos_returns_false(self):
+        config = GenerationConfig(
+            gapic_generator_version="",
+            googleapis_commitish="",
+            libraries_bom_version="",
+            template_excludes=[],
+            libraries=[library_1, library_2],
+        )
+        self.assertFalse(config.contains_common_protos())
+
     def test_validate_with_duplicate_library_name_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
@@ -131,8 +158,6 @@ def test_validate_with_duplicate_library_name_raise_exception(self):
             gapic_generator_version="",
             googleapis_commitish="",
             libraries_bom_version="",
-            owlbot_cli_image="",
-            synthtool_commitish="",
             template_excludes=[],
             libraries=[
                 LibraryConfig(
@@ -169,30 +194,6 @@ def test_from_yaml_without_googleapis_commitish_raise_exception(self):
             f"{test_config_dir}/config_without_googleapis.yaml",
         )
 
-    def test_from_yaml_without_libraries_bom_version_raise_exception(self):
-        self.assertRaisesRegex(
-            ValueError,
-            "Repo level parameter, libraries_bom_version",
-            from_yaml,
-            f"{test_config_dir}/config_without_libraries_bom_version.yaml",
-        )
-
-    def test_from_yaml_without_owlbot_cli_image_raise_exception(self):
-        self.assertRaisesRegex(
-            ValueError,
-            "Repo level parameter, owlbot_cli_image",
-            from_yaml,
-            f"{test_config_dir}/config_without_owlbot.yaml",
-        )
-
-    def test_from_yaml_without_synthtool_commitish_raise_exception(self):
-        self.assertRaisesRegex(
-            ValueError,
-            "Repo level parameter, synthtool_commitish",
-            from_yaml,
-            f"{test_config_dir}/config_without_synthtool.yaml",
-        )
-
     def test_from_yaml_without_template_excludes_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,

library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json

@@ -0,0 +1,17 @@
+{
+  "api_shortname": "baremetalsolution",
+  "name_pretty": "Bare Metal Solution",
+  "product_documentation": "https://cloud.google.com/bare-metal/docs",
+  "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.",
+  "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview",
+  "release_level": "preview",
+  "transport": "grpc",
+  "language": "java",
+  "repo": "googleapis/sdk-platform-java",
+  "repo_short": "java-bare-metal-solution",
+  "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution",
+  "library_type": "OTHER",
+  "requires_billing": true,
+  "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest",
+  "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc"
+}