Skip to content

Commit

Permalink
feat: generate proto-only repository (#2720)
Browse files Browse the repository at this point in the history
In this PR:
- Enable hermetic build script to generate `java-common-protos` and
`java-iam` in this repo.
- Change `libraries_bom_version` to optional.
- Refactor unit tests.
- Remove `java-common-protos/codecov.yaml` and `java-iam/codecov.yaml`
as they are no longer needed.
  • Loading branch information
JoeWang1127 authored and lqiu96 committed May 22, 2024
1 parent 42b8aba commit 36d358f
Show file tree
Hide file tree
Showing 13 changed files with 254 additions and 195 deletions.
4 changes: 0 additions & 4 deletions java-common-protos/codecov.yaml

This file was deleted.

1 change: 1 addition & 0 deletions java-common-protos/owlbot.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
"SECURITY.md",
"java.header",
"license-checks.xml",
"README.md",
"renovate.json",
".gitignore"
])
4 changes: 0 additions & 4 deletions java-iam/codecov.yaml

This file was deleted.

4 changes: 1 addition & 3 deletions library_generation/generate_repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ def generate_from_yaml(

for library_path, library in repo_config.libraries.items():
print(f"generating library {library.get_library_name()}")

generate_composed_library(
config_path=config_path,
config=config,
Expand All @@ -58,8 +57,7 @@ def generate_from_yaml(
versions_file=repo_config.versions_file,
)

# we skip monorepo_postprocessing if not in a monorepo
if not config.is_monorepo():
if not config.is_monorepo() or config.contains_common_protos():
return

monorepo_postprocessing(
Expand Down
2 changes: 1 addition & 1 deletion library_generation/model/gapic_inputs.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def __init__(
self,
proto_only="true",
additional_protos="google/cloud/common_resources.proto",
transport="",
transport="grpc",
rest_numeric_enum="",
gapic_yaml="",
service_config="",
Expand Down
23 changes: 17 additions & 6 deletions library_generation/model/generation_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
REPO_LEVEL_PARAMETER = "Repo level parameter"
LIBRARY_LEVEL_PARAMETER = "Library level parameter"
GAPIC_LEVEL_PARAMETER = "GAPIC level parameter"
COMMON_PROTOS_LIBRARY_NAME = "common-protos"


class GenerationConfig:
Expand All @@ -31,9 +32,9 @@ def __init__(
self,
gapic_generator_version: str,
googleapis_commitish: str,
libraries_bom_version: str,
template_excludes: list[str],
libraries: list[LibraryConfig],
libraries_bom_version: Optional[str] = None,
grpc_version: Optional[str] = None,
protoc_version: Optional[str] = None,
):
Expand All @@ -44,6 +45,9 @@ def __init__(
self.libraries = libraries
self.grpc_version = grpc_version
self.protoc_version = protoc_version
# explicit set to None so that we can compute the
# value in getter.
self.__contains_common_protos = None
self.__validate()

def get_proto_path_to_library_name(self) -> dict[str, str]:
Expand All @@ -61,6 +65,15 @@ def get_proto_path_to_library_name(self) -> dict[str, str]:
def is_monorepo(self) -> bool:
return len(self.libraries) > 1

def contains_common_protos(self) -> bool:
if self.__contains_common_protos is None:
self.__contains_common_protos = False
for library in self.libraries:
if library.get_library_name() == COMMON_PROTOS_LIBRARY_NAME:
self.__contains_common_protos = True
break
return self.__contains_common_protos

def __validate(self) -> None:
seen_library_names = dict()
for library in self.libraries:
Expand Down Expand Up @@ -133,15 +146,13 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
gapic_generator_version=__required(
config, "gapic_generator_version", REPO_LEVEL_PARAMETER
),
grpc_version=__optional(config, "grpc_version", None),
protoc_version=__optional(config, "protoc_version", None),
googleapis_commitish=__required(
config, "googleapis_commitish", REPO_LEVEL_PARAMETER
),
libraries_bom_version=__required(
config, "libraries_bom_version", REPO_LEVEL_PARAMETER
),
template_excludes=__required(config, "template_excludes", REPO_LEVEL_PARAMETER),
grpc_version=__optional(config, "grpc_version", None),
protoc_version=__optional(config, "protoc_version", None),
libraries_bom_version=__optional(config, "libraries_bom_version", None),
libraries=parsed_libraries,
)

Expand Down
1 change: 1 addition & 0 deletions library_generation/test/cli/entry_point_unit_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

class EntryPointTest(unittest.TestCase):
def test_entry_point_without_config_raise_file_exception(self):
os.chdir(script_dir)
runner = CliRunner()
# noinspection PyTypeChecker
result = runner.invoke(generate, ["--repository-path=."])
Expand Down
132 changes: 132 additions & 0 deletions library_generation/test/model/gapic_inputs_unit_tests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
import os
import unittest
from pathlib import Path

from parameterized import parameterized
from library_generation.model.gapic_inputs import parse

script_dir = os.path.dirname(os.path.realpath(__file__))
resources_dir = os.path.join(script_dir, "..", "resources")
build_file = Path(os.path.join(resources_dir, "misc")).resolve()


class UtilitiesTest(unittest.TestCase):
@parameterized.expand(
[
("BUILD_no_additional_protos.bazel", " "),
("BUILD_common_resources.bazel", " google/cloud/common_resources.proto"),
("BUILD_comment_common_resources.bazel", " "),
("BUILD_locations.bazel", " google/cloud/location/locations.proto"),
("BUILD_comment_locations.bazel", " "),
("BUILD_iam_policy.bazel", " google/iam/v1/iam_policy.proto"),
("BUILD_comment_iam_policy.bazel", " "),
(
"BUILD_iam_locations.bazel",
" google/cloud/location/locations.proto google/iam/v1/iam_policy.proto",
),
]
)
def test_gapic_inputs_parse_additional_protos(self, build_name, expected):
parsed = parse(build_file, "", build_name)
self.assertEqual(
expected,
parsed.additional_protos,
)

def test_gapic_inputs_parse_grpc_only_succeeds(self):
parsed = parse(build_file, "", "BUILD_grpc.bazel")
self.assertEqual("grpc", parsed.transport)

def test_gapic_inputs_parse_grpc_rest_succeeds(self):
parsed = parse(build_file, "", "BUILD_grpc_rest.bazel")
self.assertEqual("grpc+rest", parsed.transport)

def test_gapic_inputs_parse_rest_succeeds(self):
parsed = parse(build_file, "", "BUILD_rest.bazel")
self.assertEqual("rest", parsed.transport)

def test_gapic_inputs_parse_empty_include_samples_succeeds(self):
parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel")
self.assertEqual("false", parsed.include_samples)

def test_gapic_inputs_parse_include_samples_false_succeeds(self):
parsed = parse(build_file, "", "BUILD_include_samples_false.bazel")
self.assertEqual("false", parsed.include_samples)

def test_gapic_inputs_parse_include_samples_true_succeeds(self):
parsed = parse(build_file, "", "BUILD_include_samples_true.bazel")
self.assertEqual("true", parsed.include_samples)

def test_gapic_inputs_parse_empty_rest_numeric_enums_succeeds(self):
parsed = parse(build_file, "", "BUILD_rest_numeric_enums_empty.bazel")
self.assertEqual("false", parsed.rest_numeric_enum)

def test_gapic_inputs_parse_rest_numeric_enums_false_succeeds(self):
parsed = parse(build_file, "", "BUILD_rest_numeric_enums_false.bazel")
self.assertEqual("false", parsed.rest_numeric_enum)

def test_gapic_inputs_parse_rest_numeric_enums_true_succeeds(self):
parsed = parse(build_file, "", "BUILD_rest_numeric_enums_true.bazel")
self.assertEqual("true", parsed.rest_numeric_enum)

def test_gapic_inputs_parse_no_gapic_library_returns_proto_only_true(self):
# include_samples_empty only has a gradle assembly rule
parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel")
self.assertEqual("true", parsed.proto_only)

def test_gapic_inputs_parse_with_gapic_library_returns_proto_only_false(self):
# rest.bazel has a java_gapic_library rule
parsed = parse(build_file, "", "BUILD_rest.bazel")
self.assertEqual("false", parsed.proto_only)

def test_gapic_inputs_parse_gapic_yaml_succeeds(self):
parsed = parse(build_file, "test/versioned/path", "BUILD_gapic_yaml.bazel")
self.assertEqual("test/versioned/path/test_gapic_yaml.yaml", parsed.gapic_yaml)

def test_gapic_inputs_parse_no_gapic_yaml_returns_empty_string(self):
parsed = parse(build_file, "test/versioned/path", "BUILD_no_gapic_yaml.bazel")
self.assertEqual("", parsed.gapic_yaml)

def test_gapic_inputs_parse_service_config_succeeds(self):
parsed = parse(build_file, "test/versioned/path", "BUILD_service_config.bazel")
self.assertEqual(
"test/versioned/path/test_service_config.json", parsed.service_config
)

def test_gapic_inputs_parse_service_yaml_relative_target(self):
parsed = parse(
build_file,
"google/cloud/compute/v1",
"BUILD_service_config_relative_target.bazel",
)
self.assertEqual(
"google/cloud/compute/v1/compute_grpc_service_config.json",
parsed.service_config,
)

def test_gapic_inputs_parse_no_service_config_returns_empty_string(self):
parsed = parse(
build_file, "test/versioned/path", "BUILD_no_service_config.bazel"
)
self.assertEqual("", parsed.service_config)

def test_gapic_inputs_parse_service_yaml_succeeds(self):
parsed = parse(build_file, "test/versioned/path", "BUILD_service_yaml.bazel")
self.assertEqual(
"test/versioned/path/test_service_yaml.yaml", parsed.service_yaml
)

def test_gapic_inputs_parse_service_yaml_absolute_target(self):
parsed = parse(build_file, "", "BUILD_service_yaml_absolute_target.bazel")
self.assertEqual(
"google/cloud/videointelligence/videointelligence_v1p3beta1.yaml",
parsed.service_yaml,
)

def test_gapic_inputs_parse_no_service_yaml_returns_empty_string(self):
parsed = parse(build_file, "test/versioned/path", "BUILD_no_service_yaml.bazel")
self.assertEqual("", parsed.service_yaml)

def test_gapic_inputs_parse_proto_only_returns_grpc(self):
parsed = parse(build_file, "test/versioned/path", "BUILD_proto_only.bazel")
self.assertEqual("grpc", parsed.transport)
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@
product_documentation="",
gapic_configs=[],
)
common_protos_library = LibraryConfig(
api_shortname="common-protos",
api_description="",
name_pretty="",
product_documentation="",
gapic_configs=[],
)


class GenerationConfigTest(unittest.TestCase):
Expand Down Expand Up @@ -123,6 +130,26 @@ def test_is_monorepo_with_two_libraries_returns_true(self):
)
self.assertTrue(config.is_monorepo())

def test_contains_common_protos_with_common_protos_returns_true(self):
config = GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
template_excludes=[],
libraries=[library_1, library_2, common_protos_library],
)
self.assertTrue(config.contains_common_protos())

def test_contains_common_protos_without_common_protos_returns_false(self):
config = GenerationConfig(
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
template_excludes=[],
libraries=[library_1, library_2],
)
self.assertFalse(config.contains_common_protos())

def test_validate_with_duplicate_library_name_raise_exception(self):
self.assertRaisesRegex(
ValueError,
Expand All @@ -131,8 +158,6 @@ def test_validate_with_duplicate_library_name_raise_exception(self):
gapic_generator_version="",
googleapis_commitish="",
libraries_bom_version="",
owlbot_cli_image="",
synthtool_commitish="",
template_excludes=[],
libraries=[
LibraryConfig(
Expand Down Expand Up @@ -169,30 +194,6 @@ def test_from_yaml_without_googleapis_commitish_raise_exception(self):
f"{test_config_dir}/config_without_googleapis.yaml",
)

def test_from_yaml_without_libraries_bom_version_raise_exception(self):
self.assertRaisesRegex(
ValueError,
"Repo level parameter, libraries_bom_version",
from_yaml,
f"{test_config_dir}/config_without_libraries_bom_version.yaml",
)

def test_from_yaml_without_owlbot_cli_image_raise_exception(self):
self.assertRaisesRegex(
ValueError,
"Repo level parameter, owlbot_cli_image",
from_yaml,
f"{test_config_dir}/config_without_owlbot.yaml",
)

def test_from_yaml_without_synthtool_commitish_raise_exception(self):
self.assertRaisesRegex(
ValueError,
"Repo level parameter, synthtool_commitish",
from_yaml,
f"{test_config_dir}/config_without_synthtool.yaml",
)

def test_from_yaml_without_template_excludes_raise_exception(self):
self.assertRaisesRegex(
ValueError,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"api_shortname": "baremetalsolution",
"name_pretty": "Bare Metal Solution",
"product_documentation": "https://cloud.google.com/bare-metal/docs",
"api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.",
"client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview",
"release_level": "preview",
"transport": "grpc",
"language": "java",
"repo": "googleapis/sdk-platform-java",
"repo_short": "java-bare-metal-solution",
"distribution_name": "com.google.cloud:google-cloud-bare-metal-solution",
"library_type": "OTHER",
"requires_billing": true,
"rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest",
"rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc"
}
16 changes: 16 additions & 0 deletions library_generation/test/resources/misc/BUILD_proto_only.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
java_gapic_assembly_gradle_pkg(
name = "google-api-java",
transport = "grpc+rest",
deps = [
"annotations_proto",
"auth_proto",
"backend_proto",
"billing_proto",
"client_proto",
"config_change_proto",
"consumer_proto",
"context_proto",
"control_proto",
"distribution_proto",
],
)
Loading

0 comments on commit 36d358f

Please sign in to comment.