chore: track customization in templates (#1876)

requirements.txt

@@ -304,4 +304,4 @@ watchdog==2.1.9 \
 setuptools==65.5.1 \
     --hash=sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31 \
     --hash=sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f
-    # via -r requirements.in
+    # via -r requirements.in

synthtool/gcp/partials.py

@@ -18,12 +18,16 @@
 from typing import Dict, List
 
 # these are the default locations to look up
-_DEFAULT_PARTIAL_FILES = [".readme-partials.yml", ".readme-partials.yaml"]
+_DEFAULT_PARTIAL_FILES = [
+    ".readme-partials.yml",
+    ".readme-partials.yaml",
+    ".integration-partials.yaml",
+]
 
 
-def load_partials(files: List[str] = _DEFAULT_PARTIAL_FILES) -> Dict:
+def load_partials(files: List[str] = []) -> Dict:
     """
-    hand-crafted artisinal markdown can be provided in a .readme-partials.yml.
+    hand-crafted artisanal markdown can be provided in a .readme-partials.yml.
     The following fields are currently supported:
 
     body: custom body to include in the usage section of the document.
@@ -34,13 +38,11 @@ def load_partials(files: List[str] = _DEFAULT_PARTIAL_FILES) -> Dict:
     deprecation_warning: a warning to indicate that the library has been
         deprecated and a pointer to an alternate option
     """
+    result: Dict[str, Dict] = {}
     cwd_path = Path(os.getcwd())
-    partials_file = None
-    for file in files:
-        if os.path.exists(cwd_path / file):
-            partials_file = cwd_path / file
-            break
-    if not partials_file:
-        return {}
-    with open(partials_file) as f:
-        return yaml.load(f, Loader=yaml.SafeLoader)
+    for file in files + _DEFAULT_PARTIAL_FILES:
+        partials_file = cwd_path / file
+        if os.path.exists(partials_file):
+            with open(partials_file) as f:
+                result.update(yaml.load(f, Loader=yaml.SafeLoader))
+    return result

synthtool/gcp/templates/java_library/.kokoro/nightly/integration.cfg

@@ -35,3 +35,8 @@ env_vars: {
   key: "SECRET_MANAGER_KEYS"
   value: "java-it-service-account"
 }
+
+{% if 'partials' in metadata
+  and 'integration_append' in metadata['partials'] -%}
+{{ metadata['partials']['integration_append'] }}
+{%- endif -%}

synthtool/gcp/templates/java_library/.kokoro/nightly/java11-integration.cfg

@@ -35,3 +35,8 @@ env_vars: {
   key: "SECRET_MANAGER_KEYS"
   value: "java-it-service-account"
 }
+
+{% if 'partials' in metadata
+  and 'integration_append' in metadata['partials'] -%}
+{{ metadata['partials']['integration_append'] }}
+{%- endif -%}

synthtool/gcp/templates/java_library/.kokoro/presubmit/integration.cfg

@@ -31,3 +31,8 @@ env_vars: {
   key: "SECRET_MANAGER_KEYS"
   value: "java-it-service-account"
 }
+
+{% if 'partials' in metadata
+  and 'integration_append' in metadata['partials'] -%}
+{{ metadata['partials']['integration_append'] }}
+{%- endif -%}

synthtool/languages/java.py

@@ -472,7 +472,9 @@ def _common_template_metadata() -> Dict[str, Any]:
 
 
 def common_templates(
-    excludes: List[str] = [], template_path: Optional[Path] = None, **kwargs
+    excludes: List[str] = [],
+    template_path: Optional[Path] = None,
+    **kwargs,
 ) -> None:
     """Generate common templates for a Java Library
 
@@ -481,8 +483,9 @@ def common_templates(
     their expected location.
 
     Args:
-        excludes (List[str], optional): List of template paths to ignore
-        **kwargs: Additional options for CommonTemplates.java_library()
+        :param excludes: List of template paths to ignore
+        :param template_path:
+        :param kwargs: Additional options for CommonTemplates.java_library()
     """
     metadata = _common_template_metadata()
     kwargs["metadata"] = metadata

tests/fixtures/java_templates/defaults_test/.repo-metadata.json

@@ -0,0 +1,18 @@
+{
+    "api_shortname": "cloudasset",
+    "name_pretty": "Cloud Asset Inventory",
+    "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
+    "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
+    "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.",
+    "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html",
+    "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0",
+    "release_level": "stable",
+    "transport": "grpc",
+    "requires_billing": true,
+    "language": "java",
+    "repo": "googleapis/java-asset",
+    "repo_short": "java-asset",
+    "distribution_name": "com.google.cloud:google-cloud-asset",
+    "library_type": "GAPIC_AUTO",
+    "api_id": "cloudasset.googleapis.com"
+}

tests/fixtures/java_templates/defaults_test/java11-integration-golden.cfg

@@ -0,0 +1,38 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-public-resources/java11014"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "integration"
+}
+# TODO: remove this after we've migrated all tests and scripts
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "false"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}
+

tests/fixtures/java_templates/defaults_test/nightly-integration-golden.cfg

@@ -0,0 +1,38 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "integration"
+}
+# TODO: remove this after we've migrated all tests and scripts
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "false"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}
+

tests/fixtures/java_templates/defaults_test/presubmit-integration-golden.cfg

@@ -0,0 +1,34 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "integration"
+}
+
+# TODO: remove this after we've migrated all tests and scripts
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}
+

tests/fixtures/java_templates/partials_test/.integration-partials.yaml

@@ -0,0 +1,10 @@
+integration_append: |
+  env_vars: {
+    key: "INTEGRATION_TEST_ARGS"
+    value: "-P bigtable-emulator-it"
+  }
+  
+  env_vars: {
+    key: "GCLOUD_PROJECT"
+    value: "gcloud-devel"
+  }

tests/fixtures/java_templates/partials_test/.repo-metadata.json

@@ -0,0 +1,18 @@
+{
+    "api_shortname": "cloudasset",
+    "name_pretty": "Cloud Asset Inventory",
+    "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
+    "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
+    "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.",
+    "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html",
+    "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0",
+    "release_level": "stable",
+    "transport": "grpc",
+    "requires_billing": true,
+    "language": "java",
+    "repo": "googleapis/java-asset",
+    "repo_short": "java-asset",
+    "distribution_name": "com.google.cloud:google-cloud-asset",
+    "library_type": "GAPIC_AUTO",
+    "api_id": "cloudasset.googleapis.com"
+}

tests/fixtures/java_templates/partials_test/java11-integration-golden.cfg

@@ -0,0 +1,47 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-public-resources/java11014"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "integration"
+}
+# TODO: remove this after we've migrated all tests and scripts
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "false"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}
+
+env_vars: {
+  key: "INTEGRATION_TEST_ARGS"
+  value: "-P bigtable-emulator-it"
+}
+
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "gcloud-devel"
+}

tests/fixtures/java_templates/partials_test/nightly-integration-golden.cfg

@@ -0,0 +1,47 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "integration"
+}
+# TODO: remove this after we've migrated all tests and scripts
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "false"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}
+
+env_vars: {
+  key: "INTEGRATION_TEST_ARGS"
+  value: "-P bigtable-emulator-it"
+}
+
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "gcloud-devel"
+}