From 7597e771307b889a4c5b07f51cf9f29ed6837d69 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 17:34:59 +0000 Subject: [PATCH 01/19] feat: update cloud build config to run test --- cloudbuild-test.yaml | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index 8dc09a1e085a..8f5e85e9c5be 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -16,8 +16,9 @@ # Reduce this timeout by moving the installation of Python runtimes to a separate base image timeout: 7200s # 2 hours for the first uncached run, can be lowered later. steps: - # A single step using the Kaniko executor to build and cache + # Step 1: Build the generator image using Kaniko and push it to the registry. - name: 'gcr.io/kaniko-project/executor:latest' + id: 'build-generator' args: # Specifies the Dockerfile path - '--dockerfile=.generator/Dockerfile' @@ -27,11 +28,35 @@ steps: - '--destination=gcr.io/$PROJECT_ID/python-librarian-generator:latest' # Enables Kaniko's remote registry caching - '--cache=true' - # (Optional but recommended) Sets a time-to-live for cache layers + # Sets a time-to-live for cache layers - '--cache-ttl=24h' -# The 'images' section is no longer needed because Kaniko pushes the image itself. + # Step 2: Clone the googleapis repository into the workspace. + # This runs in parallel with the image build. + - name: 'gcr.io/cloud-builders/git' + id: 'clone-googleapis' + args: ['clone', '--depth', '1', 'https://github.com/googleapis/googleapis.git', '/workspace/googleapis'] + waitFor: ['-'] + + # Step 3: Run the generator to generate the library code. + # This step waits for the image to be built and the repo to be cloned. + - name: 'gcr.io/cloud-builders/docker' + id: 'generate-library' + args: + - 'run' + - '--rm' + # Mount the cloned googleapis repo from the workspace. + - '-v' + - '/workspace/googleapis:/app/source/googleapis' + # Mount the generate-request.json from this repo's workspace. + - '-v' + - '/workspace/.librarian/generate-request.json:/app/librarian/generate-request.json' + # The image that was built in the first step. + - 'gcr.io/$PROJECT_ID/python-librarian-generator:latest' + # The command to execute inside the container. + - 'generate' + waitFor: ['build-generator', 'clone-googleapis'] options: default_logs_bucket_behavior: REGIONAL_USER_OWNED_BUCKET - machineType: E2_HIGHCPU_32 + machineType: E2_HIGHCPU_32 \ No newline at end of file From 19a1868dbadc0ba99f461b5f63d9fbfcf7fa8890 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 17:40:55 +0000 Subject: [PATCH 02/19] update comment --- cloudbuild-test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index 8f5e85e9c5be..0c1e995a0644 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -39,7 +39,6 @@ steps: waitFor: ['-'] # Step 3: Run the generator to generate the library code. - # This step waits for the image to be built and the repo to be cloned. - name: 'gcr.io/cloud-builders/docker' id: 'generate-library' args: From 9081151f79f9246c2c083aa9e4ac96c01d98a288 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 18:28:15 +0000 Subject: [PATCH 03/19] mount librarian folder --- cloudbuild-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index 0c1e995a0644..4656a5f3feaf 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -49,7 +49,7 @@ steps: - '/workspace/googleapis:/app/source/googleapis' # Mount the generate-request.json from this repo's workspace. - '-v' - - '/workspace/.librarian/generate-request.json:/app/librarian/generate-request.json' + - '/workspace/.librarian:/app/librarian' # The image that was built in the first step. - 'gcr.io/$PROJECT_ID/python-librarian-generator:latest' # The command to execute inside the container. From adef3f48ce9af5b8d3625d0dca60f218f325f901 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 19:00:40 +0000 Subject: [PATCH 04/19] add test config --- .generator/test-resources/generate-request.json | 0 cloudbuild-test.yaml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 .generator/test-resources/generate-request.json diff --git a/.generator/test-resources/generate-request.json b/.generator/test-resources/generate-request.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index 4656a5f3feaf..ab6d8b1529b8 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -49,7 +49,7 @@ steps: - '/workspace/googleapis:/app/source/googleapis' # Mount the generate-request.json from this repo's workspace. - '-v' - - '/workspace/.librarian:/app/librarian' + - '/workspace/.generator/test-resources/generate-request.json:/app/librarian/generate-request.json' # The image that was built in the first step. - 'gcr.io/$PROJECT_ID/python-librarian-generator:latest' # The command to execute inside the container. From 3ff19ed6eb363368a6331f6221401a88cacb7ea9 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 19:12:43 +0000 Subject: [PATCH 05/19] populate generate request --- .../test-resources/generate-request.json | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.generator/test-resources/generate-request.json b/.generator/test-resources/generate-request.json index e69de29bb2d1..c6bfa89a1f77 100644 --- a/.generator/test-resources/generate-request.json +++ b/.generator/test-resources/generate-request.json @@ -0,0 +1,39 @@ +{ + "id": "google-cloud-language", + "version": "2.17.2", + "last_generated_commit": "97a83d76a09a7f6dcab43675c87bdfeb5bcf1cb5", + "apis": [ + { + "path": "google/cloud/language/v1beta2", + "service_config": "language_v1beta2.yaml", + "status": "" + }, + { + "path": "google/cloud/language/v2", + "service_config": "language_v2.yaml", + "status": "" + }, + { + "path": "google/cloud/language/v1", + "service_config": "language_v1.yaml", + "status": "" + } + ], + "source_roots": [ + "packages/google-cloud-language" + ], + "preserve_regex": [ + ".OwlBot.yaml", + "CHANGELOG.md", + "docs/CHANGELOG.md", + "docs/README.rst", + "samples/README.txt", + "tar.gz", + "gapic_version.py", + "samples/generated_samples/snippet_metadata_", + "scripts/client-post-processing" + ], + "remove_regex": [ + "^packages/google-cloud-language" + ] +} \ No newline at end of file From 6d304f732a8e11225b2ed76edfb16f25be21879b Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 19:22:45 +0000 Subject: [PATCH 06/19] add source as googleapis --- cloudbuild-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index ab6d8b1529b8..c61780b9ae90 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -46,7 +46,7 @@ steps: - '--rm' # Mount the cloned googleapis repo from the workspace. - '-v' - - '/workspace/googleapis:/app/source/googleapis' + - '/workspace/googleapis:/app/source' # Mount the generate-request.json from this repo's workspace. - '-v' - '/workspace/.generator/test-resources/generate-request.json:/app/librarian/generate-request.json' From 362aaf1537655bcf5c5dfd5938f7da141ef16b76 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 19:35:11 +0000 Subject: [PATCH 07/19] mount input folder --- cloudbuild-test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index c61780b9ae90..155bb4bbb497 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -50,6 +50,9 @@ steps: # Mount the generate-request.json from this repo's workspace. - '-v' - '/workspace/.generator/test-resources/generate-request.json:/app/librarian/generate-request.json' + # Mount the generator-input from this repo's workspace. + - '-v' + - '/workspace/.librarian/generator-input:/app/input' # The image that was built in the first step. - 'gcr.io/$PROJECT_ID/python-librarian-generator:latest' # The command to execute inside the container. From a6d56ef336c4638301dd50ef31ae18bf97f3a8f2 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 20:15:26 +0000 Subject: [PATCH 08/19] temporary silence .nox cleanup --- .generator/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.generator/cli.py b/.generator/cli.py index 9affe2b44c35..0e9e6a6a52c5 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -290,7 +290,7 @@ def _clean_up_files_after_post_processing(output: str, library_id: str): library_id(str): The library id to be used for post processing. """ path_to_library = f"packages/{library_id}" - shutil.rmtree(f"{output}/{path_to_library}/.nox") + # shutil.rmtree(f"{output}/{path_to_library}/.nox") os.remove(f"{output}/{path_to_library}/CHANGELOG.md") os.remove(f"{output}/{path_to_library}/docs/CHANGELOG.md") os.remove(f"{output}/{path_to_library}/docs/README.rst") From 289494f6ef52563e421aae547c0fb8073d15342c Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 20:53:35 +0000 Subject: [PATCH 09/19] update clean up function --- .generator/cli.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/.generator/cli.py b/.generator/cli.py index 0e9e6a6a52c5..fce893a6c270 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -280,9 +280,16 @@ def _copy_files_needed_for_post_processing(output: str, input: str, library_id: ) +import glob +import os +import shutil +from pathlib import Path + + def _clean_up_files_after_post_processing(output: str, library_id: str): """ - Clean up files which should not be included in the generated client + Clean up files which should not be included in the generated client. + This function is idempotent and will not fail if files are already removed. Args: output(str): Path to the directory in the container where code @@ -290,23 +297,31 @@ def _clean_up_files_after_post_processing(output: str, library_id: str): library_id(str): The library id to be used for post processing. """ path_to_library = f"packages/{library_id}" - # shutil.rmtree(f"{output}/{path_to_library}/.nox") - os.remove(f"{output}/{path_to_library}/CHANGELOG.md") - os.remove(f"{output}/{path_to_library}/docs/CHANGELOG.md") - os.remove(f"{output}/{path_to_library}/docs/README.rst") + + # Safely remove directories, ignoring errors if they don't exist. + shutil.rmtree(f"{output}/{path_to_library}/.nox", ignore_errors=True) + shutil.rmtree(f"{output}/owl-bot-staging", ignore_errors=True) + + # Safely remove specific files if they exist using pathlib. + Path(f"{output}/{path_to_library}/CHANGELOG.md").unlink(missing_ok=True) + Path(f"{output}/{path_to_library}/docs/CHANGELOG.md").unlink(missing_ok=True) + Path(f"{output}/{path_to_library}/docs/README.rst").unlink(missing_ok=True) + + # The glob loops are already safe, as they do nothing if no files match. for post_processing_file in glob.glob( f"{output}/{path_to_library}/scripts/client-post-processing/*.yaml" ): # pragma: NO COVER os.remove(post_processing_file) + for gapic_version_file in glob.glob( f"{output}/{path_to_library}/**/gapic_version.py", recursive=True ): # pragma: NO COVER os.remove(gapic_version_file) + for snippet_metadata_file in glob.glob( f"{output}/{path_to_library}/samples/generated_samples/snippet_metadata*.json" ): # pragma: NO COVER os.remove(snippet_metadata_file) - shutil.rmtree(f"{output}/owl-bot-staging") def handle_generate( From 662fbcf5f108560e0e696758e322917d513b7c66 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 21:07:19 +0000 Subject: [PATCH 10/19] add generate response file --- .generator/test-resources/generate-response.json | 0 cloudbuild-test.yaml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 .generator/test-resources/generate-response.json diff --git a/.generator/test-resources/generate-response.json b/.generator/test-resources/generate-response.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index 155bb4bbb497..90547b39b557 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -49,7 +49,7 @@ steps: - '/workspace/googleapis:/app/source' # Mount the generate-request.json from this repo's workspace. - '-v' - - '/workspace/.generator/test-resources/generate-request.json:/app/librarian/generate-request.json' + - '/workspace/.generator/test-resources:/app/librarian' # Mount the generator-input from this repo's workspace. - '-v' - '/workspace/.librarian/generator-input:/app/input' From 51b2c550dd69be49a878fdbfabe277081dbb0a55 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 27 Aug 2025 23:08:48 +0000 Subject: [PATCH 11/19] create writable dir --- cloudbuild-test.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index 90547b39b557..8a79d0e65435 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -38,6 +38,18 @@ steps: args: ['clone', '--depth', '1', 'https://github.com/googleapis/googleapis.git', '/workspace/googleapis'] waitFor: ['-'] + # Step 3: Prepare a writable directory for I/O. + # This step creates a new directory and copies the input file into it. + - name: 'alpine' + id: 'prepare-io-dir' + args: + - 'sh' + - '-c' + - | + mkdir -p /workspace/librarian-io && \ + cp /workspace/.generator/test-resources/generate-request.json /workspace/librarian-io/ + waitFor: ['-'] + # Step 3: Run the generator to generate the library code. - name: 'gcr.io/cloud-builders/docker' id: 'generate-library' @@ -49,7 +61,7 @@ steps: - '/workspace/googleapis:/app/source' # Mount the generate-request.json from this repo's workspace. - '-v' - - '/workspace/.generator/test-resources:/app/librarian' + - '/workspace/librarian-io:/app/librarian' # Mount the generator-input from this repo's workspace. - '-v' - '/workspace/.librarian/generator-input:/app/input' From 68e2efcf7de1a2bb1c915100bae1b83ecf6eaf95 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Thu, 28 Aug 2025 16:17:02 +0000 Subject: [PATCH 12/19] remove test generate response --- .generator/test-resources/generate-response.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .generator/test-resources/generate-response.json diff --git a/.generator/test-resources/generate-response.json b/.generator/test-resources/generate-response.json deleted file mode 100644 index e69de29bb2d1..000000000000 From 583617d41a54a6f28ea93c4ca5653d4b40e1290f Mon Sep 17 00:00:00 2001 From: ohmayr Date: Thu, 28 Aug 2025 17:07:19 +0000 Subject: [PATCH 13/19] revert back to mounting test-resources --- cloudbuild-test.yaml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index 8a79d0e65435..244bee76c72a 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -38,18 +38,6 @@ steps: args: ['clone', '--depth', '1', 'https://github.com/googleapis/googleapis.git', '/workspace/googleapis'] waitFor: ['-'] - # Step 3: Prepare a writable directory for I/O. - # This step creates a new directory and copies the input file into it. - - name: 'alpine' - id: 'prepare-io-dir' - args: - - 'sh' - - '-c' - - | - mkdir -p /workspace/librarian-io && \ - cp /workspace/.generator/test-resources/generate-request.json /workspace/librarian-io/ - waitFor: ['-'] - # Step 3: Run the generator to generate the library code. - name: 'gcr.io/cloud-builders/docker' id: 'generate-library' @@ -65,6 +53,9 @@ steps: # Mount the generator-input from this repo's workspace. - '-v' - '/workspace/.librarian/generator-input:/app/input' + # Mount the test-resources from this repo's workspace as the librarian dir. + - '-v' + - '/workspace/.generator/test-resources:/app/librarian' # The image that was built in the first step. - 'gcr.io/$PROJECT_ID/python-librarian-generator:latest' # The command to execute inside the container. From 8392e79bb6556a9a73ea4bd9bc9eee5fa0ee6ae9 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Thu, 28 Aug 2025 17:14:33 +0000 Subject: [PATCH 14/19] add librarian folder --- .../test-resources/{ => librarian}/generate-request.json | 0 cloudbuild-test.yaml | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename .generator/test-resources/{ => librarian}/generate-request.json (100%) diff --git a/.generator/test-resources/generate-request.json b/.generator/test-resources/librarian/generate-request.json similarity index 100% rename from .generator/test-resources/generate-request.json rename to .generator/test-resources/librarian/generate-request.json diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index 244bee76c72a..a6a47c983e12 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -53,9 +53,9 @@ steps: # Mount the generator-input from this repo's workspace. - '-v' - '/workspace/.librarian/generator-input:/app/input' - # Mount the test-resources from this repo's workspace as the librarian dir. + # Mount the test-resources/librarian from this repo's workspace as the librarian dir. - '-v' - - '/workspace/.generator/test-resources:/app/librarian' + - '/workspace/.generator/test-resources/librarian:/app/librarian' # The image that was built in the first step. - 'gcr.io/$PROJECT_ID/python-librarian-generator:latest' # The command to execute inside the container. From 1e09ecd9cd22395a41782a074f8de46a91f27d43 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Thu, 28 Aug 2025 17:17:05 +0000 Subject: [PATCH 15/19] fix state.yaml --- .generator/test-resources/librarian/generate-request.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.generator/test-resources/librarian/generate-request.json b/.generator/test-resources/librarian/generate-request.json index c6bfa89a1f77..cc71328e7b54 100644 --- a/.generator/test-resources/librarian/generate-request.json +++ b/.generator/test-resources/librarian/generate-request.json @@ -34,6 +34,6 @@ "scripts/client-post-processing" ], "remove_regex": [ - "^packages/google-cloud-language" + "packages/google-cloud-language" ] } \ No newline at end of file From 051a0c5570efdf6ce2702511c84abec9b299d534 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Thu, 28 Aug 2025 17:32:23 +0000 Subject: [PATCH 16/19] update generate request --- .generator/test-resources/librarian/generate-request.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.generator/test-resources/librarian/generate-request.json b/.generator/test-resources/librarian/generate-request.json index cc71328e7b54..c5ae4f62b9de 100644 --- a/.generator/test-resources/librarian/generate-request.json +++ b/.generator/test-resources/librarian/generate-request.json @@ -24,7 +24,7 @@ ], "preserve_regex": [ ".OwlBot.yaml", - "CHANGELOG.md", + "packages/google-cloud-language/CHANGELOG.md", "docs/CHANGELOG.md", "docs/README.rst", "samples/README.txt", From c9504d49e20679555a6fc81306b2f50d28674527 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Thu, 28 Aug 2025 17:34:27 +0000 Subject: [PATCH 17/19] fix duplicate mount --- cloudbuild-test.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index a6a47c983e12..4f91d58fda7e 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -47,9 +47,6 @@ steps: # Mount the cloned googleapis repo from the workspace. - '-v' - '/workspace/googleapis:/app/source' - # Mount the generate-request.json from this repo's workspace. - - '-v' - - '/workspace/librarian-io:/app/librarian' # Mount the generator-input from this repo's workspace. - '-v' - '/workspace/.librarian/generator-input:/app/input' From f6832ffb3e3e98ae67e706fc8a7539dc0ba75132 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Thu, 28 Aug 2025 17:35:22 +0000 Subject: [PATCH 18/19] fix newline --- cloudbuild-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index 4f91d58fda7e..53767c7411f6 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -61,4 +61,4 @@ steps: options: default_logs_bucket_behavior: REGIONAL_USER_OWNED_BUCKET - machineType: E2_HIGHCPU_32 \ No newline at end of file + machineType: E2_HIGHCPU_32 From 2ffa2a86c7977dbdd3dbb7b6db3f4a94a17c2335 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Thu, 28 Aug 2025 17:45:34 +0000 Subject: [PATCH 19/19] fix imports --- .generator/cli.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.generator/cli.py b/.generator/cli.py index fce893a6c270..4473748fd4d8 100644 --- a/.generator/cli.py +++ b/.generator/cli.py @@ -21,6 +21,7 @@ import shutil import subprocess import sys +from pathlib import Path from typing import Dict, List try: @@ -280,12 +281,6 @@ def _copy_files_needed_for_post_processing(output: str, input: str, library_id: ) -import glob -import os -import shutil -from pathlib import Path - - def _clean_up_files_after_post_processing(output: str, library_id: str): """ Clean up files which should not be included in the generated client.