From a79241941a00783e0da86853c455e899567c3ccb Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Tue, 17 Oct 2023 17:31:44 +0000 Subject: [PATCH] chore: track customization in templates (#1876) --- requirements.txt | 2 +- synthtool/gcp/partials.py | 26 +++++----- .../.kokoro/nightly/integration.cfg | 5 ++ .../.kokoro/nightly/java11-integration.cfg | 5 ++ .../.kokoro/presubmit/integration.cfg | 5 ++ synthtool/languages/java.py | 9 ++-- .../defaults_test/.repo-metadata.json | 18 +++++++ .../java11-integration-golden.cfg | 38 +++++++++++++++ .../nightly-integration-golden.cfg | 38 +++++++++++++++ .../presubmit-integration-golden.cfg | 34 ++++++++++++++ .../partials_test/.integration-partials.yaml | 10 ++++ .../partials_test/.repo-metadata.json | 18 +++++++ .../java11-integration-golden.cfg | 47 +++++++++++++++++++ .../nightly-integration-golden.cfg | 47 +++++++++++++++++++ .../presubmit-integration-golden.cfg | 43 +++++++++++++++++ tests/test_language_java.py | 34 ++++++++++++++ 16 files changed, 363 insertions(+), 16 deletions(-) create mode 100644 tests/fixtures/java_templates/defaults_test/.repo-metadata.json create mode 100644 tests/fixtures/java_templates/defaults_test/java11-integration-golden.cfg create mode 100644 tests/fixtures/java_templates/defaults_test/nightly-integration-golden.cfg create mode 100644 tests/fixtures/java_templates/defaults_test/presubmit-integration-golden.cfg create mode 100644 tests/fixtures/java_templates/partials_test/.integration-partials.yaml create mode 100644 tests/fixtures/java_templates/partials_test/.repo-metadata.json create mode 100644 tests/fixtures/java_templates/partials_test/java11-integration-golden.cfg create mode 100644 tests/fixtures/java_templates/partials_test/nightly-integration-golden.cfg create mode 100644 tests/fixtures/java_templates/partials_test/presubmit-integration-golden.cfg diff --git a/requirements.txt b/requirements.txt index d7cb2592f..539bcb8b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -304,4 +304,4 @@ watchdog==2.1.9 \ setuptools==65.5.1 \ --hash=sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31 \ --hash=sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f - # via -r requirements.in + # via -r requirements.in \ No newline at end of file diff --git a/synthtool/gcp/partials.py b/synthtool/gcp/partials.py index 59f5b86f4..da164cf9b 100644 --- a/synthtool/gcp/partials.py +++ b/synthtool/gcp/partials.py @@ -18,12 +18,16 @@ from typing import Dict, List # these are the default locations to look up -_DEFAULT_PARTIAL_FILES = [".readme-partials.yml", ".readme-partials.yaml"] +_DEFAULT_PARTIAL_FILES = [ + ".readme-partials.yml", + ".readme-partials.yaml", + ".integration-partials.yaml", +] -def load_partials(files: List[str] = _DEFAULT_PARTIAL_FILES) -> Dict: +def load_partials(files: List[str] = []) -> Dict: """ - hand-crafted artisinal markdown can be provided in a .readme-partials.yml. + hand-crafted artisanal markdown can be provided in a .readme-partials.yml. The following fields are currently supported: body: custom body to include in the usage section of the document. @@ -34,13 +38,11 @@ def load_partials(files: List[str] = _DEFAULT_PARTIAL_FILES) -> Dict: deprecation_warning: a warning to indicate that the library has been deprecated and a pointer to an alternate option """ + result: Dict[str, Dict] = {} cwd_path = Path(os.getcwd()) - partials_file = None - for file in files: - if os.path.exists(cwd_path / file): - partials_file = cwd_path / file - break - if not partials_file: - return {} - with open(partials_file) as f: - return yaml.load(f, Loader=yaml.SafeLoader) + for file in files + _DEFAULT_PARTIAL_FILES: + partials_file = cwd_path / file + if os.path.exists(partials_file): + with open(partials_file) as f: + result.update(yaml.load(f, Loader=yaml.SafeLoader)) + return result diff --git a/synthtool/gcp/templates/java_library/.kokoro/nightly/integration.cfg b/synthtool/gcp/templates/java_library/.kokoro/nightly/integration.cfg index 8944ac33e..46e676027 100644 --- a/synthtool/gcp/templates/java_library/.kokoro/nightly/integration.cfg +++ b/synthtool/gcp/templates/java_library/.kokoro/nightly/integration.cfg @@ -35,3 +35,8 @@ env_vars: { key: "SECRET_MANAGER_KEYS" value: "java-it-service-account" } + +{% if 'partials' in metadata + and 'integration_append' in metadata['partials'] -%} +{{ metadata['partials']['integration_append'] }} +{%- endif -%} diff --git a/synthtool/gcp/templates/java_library/.kokoro/nightly/java11-integration.cfg b/synthtool/gcp/templates/java_library/.kokoro/nightly/java11-integration.cfg index 2981759c1..0e8d1a941 100644 --- a/synthtool/gcp/templates/java_library/.kokoro/nightly/java11-integration.cfg +++ b/synthtool/gcp/templates/java_library/.kokoro/nightly/java11-integration.cfg @@ -35,3 +35,8 @@ env_vars: { key: "SECRET_MANAGER_KEYS" value: "java-it-service-account" } + +{% if 'partials' in metadata + and 'integration_append' in metadata['partials'] -%} +{{ metadata['partials']['integration_append'] }} +{%- endif -%} diff --git a/synthtool/gcp/templates/java_library/.kokoro/presubmit/integration.cfg b/synthtool/gcp/templates/java_library/.kokoro/presubmit/integration.cfg index dded67a9d..fcf3d08bf 100644 --- a/synthtool/gcp/templates/java_library/.kokoro/presubmit/integration.cfg +++ b/synthtool/gcp/templates/java_library/.kokoro/presubmit/integration.cfg @@ -31,3 +31,8 @@ env_vars: { key: "SECRET_MANAGER_KEYS" value: "java-it-service-account" } + +{% if 'partials' in metadata + and 'integration_append' in metadata['partials'] -%} +{{ metadata['partials']['integration_append'] }} +{%- endif -%} diff --git a/synthtool/languages/java.py b/synthtool/languages/java.py index 008274092..3ae198c09 100644 --- a/synthtool/languages/java.py +++ b/synthtool/languages/java.py @@ -472,7 +472,9 @@ def _common_template_metadata() -> Dict[str, Any]: def common_templates( - excludes: List[str] = [], template_path: Optional[Path] = None, **kwargs + excludes: List[str] = [], + template_path: Optional[Path] = None, + **kwargs, ) -> None: """Generate common templates for a Java Library @@ -481,8 +483,9 @@ def common_templates( their expected location. Args: - excludes (List[str], optional): List of template paths to ignore - **kwargs: Additional options for CommonTemplates.java_library() + :param excludes: List of template paths to ignore + :param template_path: + :param kwargs: Additional options for CommonTemplates.java_library() """ metadata = _common_template_metadata() kwargs["metadata"] = metadata diff --git a/tests/fixtures/java_templates/defaults_test/.repo-metadata.json b/tests/fixtures/java_templates/defaults_test/.repo-metadata.json new file mode 100644 index 000000000..840e69ca7 --- /dev/null +++ b/tests/fixtures/java_templates/defaults_test/.repo-metadata.json @@ -0,0 +1,18 @@ +{ + "api_shortname": "cloudasset", + "name_pretty": "Cloud Asset Inventory", + "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", + "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html", + "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", + "release_level": "stable", + "transport": "grpc", + "requires_billing": true, + "language": "java", + "repo": "googleapis/java-asset", + "repo_short": "java-asset", + "distribution_name": "com.google.cloud:google-cloud-asset", + "library_type": "GAPIC_AUTO", + "api_id": "cloudasset.googleapis.com" +} \ No newline at end of file diff --git a/tests/fixtures/java_templates/defaults_test/java11-integration-golden.cfg b/tests/fixtures/java_templates/defaults_test/java11-integration-golden.cfg new file mode 100644 index 000000000..5d14a782d --- /dev/null +++ b/tests/fixtures/java_templates/defaults_test/java11-integration-golden.cfg @@ -0,0 +1,38 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-public-resources/java11014" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration" +} +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "ENABLE_FLAKYBOT" + value: "false" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + diff --git a/tests/fixtures/java_templates/defaults_test/nightly-integration-golden.cfg b/tests/fixtures/java_templates/defaults_test/nightly-integration-golden.cfg new file mode 100644 index 000000000..f1ccba78f --- /dev/null +++ b/tests/fixtures/java_templates/defaults_test/nightly-integration-golden.cfg @@ -0,0 +1,38 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java8" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration" +} +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "java-docs-samples-testing" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "java-docs-samples-testing" +} + +env_vars: { + key: "ENABLE_FLAKYBOT" + value: "false" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + diff --git a/tests/fixtures/java_templates/defaults_test/presubmit-integration-golden.cfg b/tests/fixtures/java_templates/defaults_test/presubmit-integration-golden.cfg new file mode 100644 index 000000000..5864c603e --- /dev/null +++ b/tests/fixtures/java_templates/defaults_test/presubmit-integration-golden.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java8" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + diff --git a/tests/fixtures/java_templates/partials_test/.integration-partials.yaml b/tests/fixtures/java_templates/partials_test/.integration-partials.yaml new file mode 100644 index 000000000..bbc59eabd --- /dev/null +++ b/tests/fixtures/java_templates/partials_test/.integration-partials.yaml @@ -0,0 +1,10 @@ +integration_append: | + env_vars: { + key: "INTEGRATION_TEST_ARGS" + value: "-P bigtable-emulator-it" + } + + env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" + } diff --git a/tests/fixtures/java_templates/partials_test/.repo-metadata.json b/tests/fixtures/java_templates/partials_test/.repo-metadata.json new file mode 100644 index 000000000..840e69ca7 --- /dev/null +++ b/tests/fixtures/java_templates/partials_test/.repo-metadata.json @@ -0,0 +1,18 @@ +{ + "api_shortname": "cloudasset", + "name_pretty": "Cloud Asset Inventory", + "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", + "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html", + "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", + "release_level": "stable", + "transport": "grpc", + "requires_billing": true, + "language": "java", + "repo": "googleapis/java-asset", + "repo_short": "java-asset", + "distribution_name": "com.google.cloud:google-cloud-asset", + "library_type": "GAPIC_AUTO", + "api_id": "cloudasset.googleapis.com" +} \ No newline at end of file diff --git a/tests/fixtures/java_templates/partials_test/java11-integration-golden.cfg b/tests/fixtures/java_templates/partials_test/java11-integration-golden.cfg new file mode 100644 index 000000000..b83c0b7e8 --- /dev/null +++ b/tests/fixtures/java_templates/partials_test/java11-integration-golden.cfg @@ -0,0 +1,47 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-public-resources/java11014" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration" +} +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "ENABLE_FLAKYBOT" + value: "false" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + +env_vars: { + key: "INTEGRATION_TEST_ARGS" + value: "-P bigtable-emulator-it" +} + +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} diff --git a/tests/fixtures/java_templates/partials_test/nightly-integration-golden.cfg b/tests/fixtures/java_templates/partials_test/nightly-integration-golden.cfg new file mode 100644 index 000000000..85c9fdae3 --- /dev/null +++ b/tests/fixtures/java_templates/partials_test/nightly-integration-golden.cfg @@ -0,0 +1,47 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java8" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration" +} +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "java-docs-samples-testing" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "java-docs-samples-testing" +} + +env_vars: { + key: "ENABLE_FLAKYBOT" + value: "false" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + +env_vars: { + key: "INTEGRATION_TEST_ARGS" + value: "-P bigtable-emulator-it" +} + +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} diff --git a/tests/fixtures/java_templates/partials_test/presubmit-integration-golden.cfg b/tests/fixtures/java_templates/partials_test/presubmit-integration-golden.cfg new file mode 100644 index 000000000..f2a119bdc --- /dev/null +++ b/tests/fixtures/java_templates/partials_test/presubmit-integration-golden.cfg @@ -0,0 +1,43 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java8" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + +env_vars: { + key: "INTEGRATION_TEST_ARGS" + value: "-P bigtable-emulator-it" +} + +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} diff --git a/tests/test_language_java.py b/tests/test_language_java.py index f98602c92..30bf86da4 100644 --- a/tests/test_language_java.py +++ b/tests/test_language_java.py @@ -254,6 +254,40 @@ def test_release_please_handle_releases(): ) +def test_defaults(): + with util.copied_fixtures_dir(FIXTURES / "java_templates" / "defaults_test"): + java.common_templates(template_path=TEMPLATES_PATH) + assert os.path.isfile(".kokoro/nightly/integration.cfg") + assert_matches_golden( + "nightly-integration-golden.cfg", ".kokoro/nightly/integration.cfg" + ) + assert os.path.isfile(".kokoro/nightly/java11-integration.cfg") + assert_matches_golden( + "java11-integration-golden.cfg", ".kokoro/nightly/java11-integration.cfg" + ) + assert os.path.isfile(".kokoro/presubmit/integration.cfg") + assert_matches_golden( + "presubmit-integration-golden.cfg", ".kokoro/presubmit/integration.cfg" + ) + + +def test_merge_partials(): + with util.copied_fixtures_dir(FIXTURES / "java_templates" / "partials_test"): + java.common_templates(template_path=TEMPLATES_PATH) + assert os.path.isfile(".kokoro/nightly/integration.cfg") + assert_matches_golden( + "nightly-integration-golden.cfg", ".kokoro/nightly/integration.cfg" + ) + assert os.path.isfile(".kokoro/nightly/java11-integration.cfg") + assert_matches_golden( + "java11-integration-golden.cfg", ".kokoro/nightly/java11-integration.cfg" + ) + assert os.path.isfile(".kokoro/presubmit/integration.cfg") + assert_matches_golden( + "presubmit-integration-golden.cfg", ".kokoro/presubmit/integration.cfg" + ) + + def assert_matches_golden(expected, actual): matching_lines = 0 with open(actual, "rt") as fp: