diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 6b6b90302c..d0f61d518e 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -18,7 +18,8 @@ branchProtectionRules: - 'Kokoro - Test: Java GraalVM Native Image' - 'Kokoro - Test: Java 17 GraalVM Native Image' - javadoc - - unmanaged_dependency_check +# TODO: re-enabled once https://github.com/googleapis/sdk-platform-java/pull/3078 is merged and released +# - unmanaged_dependency_check - pattern: 1.113.14-sp isAdminEnforced: true requiredApprovingReviewCount: 1 diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index cc49d69ff4..8b479f3a45 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -16,12 +16,13 @@ name: Hermetic library generation upon generation config change through pull requests on: pull_request: - paths: - - 'generation_config.yaml' + +env: + HEAD_REF: ${{ github.head_ref }} jobs: library_generation: - # skip pull requests coming from a forked repository + # skip pull requests come from a forked repository if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest steps: @@ -37,6 +38,6 @@ jobs: [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" bash .github/scripts/hermetic_library_generation.sh \ --target_branch ${{ github.base_ref }} \ - --current_branch ${{ github.head_ref }} + --current_branch $HEAD_REF env: GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/.github/workflows/unmanaged_dependency_check.yaml b/.github/workflows/unmanaged_dependency_check.yaml index aa984df018..35eacb9ba3 100644 --- a/.github/workflows/unmanaged_dependency_check.yaml +++ b/.github/workflows/unmanaged_dependency_check.yaml @@ -17,6 +17,6 @@ jobs: # repository .kokoro/build.sh - name: Unmanaged dependency check - uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.32.0 + uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.33.0 with: bom-path: google-cloud-storage-bom/pom.xml diff --git a/.kokoro/presubmit/graalvm-native-17.cfg b/.kokoro/presubmit/graalvm-native-17.cfg index 00b7922115..1b1612f6d5 100644 --- a/.kokoro/presubmit/graalvm-native-17.cfg +++ b/.kokoro/presubmit/graalvm-native-17.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.32.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.33.0" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native.cfg b/.kokoro/presubmit/graalvm-native.cfg index b16171fd67..8da4fbaf49 100644 --- a/.kokoro/presubmit/graalvm-native.cfg +++ b/.kokoro/presubmit/graalvm-native.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.32.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.33.0" } env_vars: { diff --git a/.readme-partials.yaml b/.readme-partials.yaml index adc8597c84..528566756f 100644 --- a/.readme-partials.yaml +++ b/.readme-partials.yaml @@ -6,13 +6,14 @@ custom_content: | The Storage Control API creates one space to perform metadata-specific, control plane, and long-running operations apart from the Storage API. Separating these operations from the Storage API improves API standardization and lets you run faster releases. If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + ```xml com.google.cloud libraries-bom - 26.37.0 + 26.44.0 pom import @@ -21,30 +22,33 @@ custom_content: | - com.google.cloud - google-cloud-storage-control - + com.google.cloud + google-cloud-storage-control + + ``` If you are using Maven without the BOM, add this to your dependencies: - + + + ```xml com.google.cloud google-cloud-storage-control - 2.40.2-SNAPSHOT + 2.41.0 ``` If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy - implementation platform('com.google.cloud:libraries-bom:2.40.1') + implementation platform('com.google.cloud:libraries-bom:26.44.0') implementation 'com.google.cloud:google-cloud-storage-control' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy - implementation 'com.google.cloud:google-cloud-storage-control:2.40.2-SNAPSHOT' + implementation 'com.google.cloud:google-cloud-storage-control:2.41.0' ``` #### Creating an authorized service object diff --git a/.repo-metadata.json b/.repo-metadata.json index bc02423b9e..18fe4b0c5a 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -5,7 +5,7 @@ "api_description": "is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally.", "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-storage/latest/history", "release_level": "stable", - "transport": "grpc", + "transport": "http", "language": "java", "repo": "googleapis/java-storage", "repo_short": "java-storage", diff --git a/CHANGELOG.md b/CHANGELOG.md index 5821bc840d..a889954b2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [2.41.0](https://github.com/googleapis/java-storage/compare/v2.40.1...v2.41.0) (2024-07-31) + + +### Features + +* Enable gRPC client open telemetry metrics reporting ([#2590](https://github.com/googleapis/java-storage/issues/2590)) ([d153228](https://github.com/googleapis/java-storage/commit/d153228a301007b5952de9722f370dda0784473a)) + + +### Bug Fixes + +* Add UnknownHostException to set of retriable exception ([#2651](https://github.com/googleapis/java-storage/issues/2651)) ([18de9fc](https://github.com/googleapis/java-storage/commit/18de9fcdb831132336eca4112dfe0515174bba7b)) +* Update grpc resumable upload error categorization to be more tolerant ([#2644](https://github.com/googleapis/java-storage/issues/2644)) ([95697dd](https://github.com/googleapis/java-storage/commit/95697dd3d744351058c13793c6ae576820f6b638)) +* Update Storage#readAllBytes to respect shouldReturnRawInputStream option ([#2635](https://github.com/googleapis/java-storage/issues/2635)) ([dc883cc](https://github.com/googleapis/java-storage/commit/dc883cce5f547def7cfb34c4f8a2d409493e4cb9)) +* Update TransferManager downloads to reduce in memory buffering ([#2630](https://github.com/googleapis/java-storage/issues/2630)) ([fc2fd75](https://github.com/googleapis/java-storage/commit/fc2fd750ed60b840e6285a4b1f4ecce739df4c09)) +* Use fast calculation for totalRemaining number of bytes from multiple ByteBuffers ([#2633](https://github.com/googleapis/java-storage/issues/2633)) ([758b3dd](https://github.com/googleapis/java-storage/commit/758b3dd3cc4f6dfc2dfc12c3a77472d97c31c5d5)) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240625-2.0.0 ([#2616](https://github.com/googleapis/java-storage/issues/2616)) ([b22babb](https://github.com/googleapis/java-storage/commit/b22babbe26572d8c4289a65a0b125b2a60e8ef79)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240706-2.0.0 ([#2634](https://github.com/googleapis/java-storage/issues/2634)) ([1ccaa0c](https://github.com/googleapis/java-storage/commit/1ccaa0c64887a0661438957e9427237ee005ccf1)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.33.0 ([#2647](https://github.com/googleapis/java-storage/issues/2647)) ([8196259](https://github.com/googleapis/java-storage/commit/8196259927330ecfe3e604c24d248f7935e7fe0d)) +* Update dependency net.jqwik:jqwik to v1.9.0 ([#2608](https://github.com/googleapis/java-storage/issues/2608)) ([a20eb66](https://github.com/googleapis/java-storage/commit/a20eb660ddfa4b68d79ce04496064f3025676d5a)) +* Update dependency org.junit.vintage:junit-vintage-engine to v5.10.3 ([#2604](https://github.com/googleapis/java-storage/issues/2604)) ([8c79f39](https://github.com/googleapis/java-storage/commit/8c79f39ad78d100065c189bcf8e18644b29ff9ed)) +* Update junit-platform.version to v5.10.3 ([#2605](https://github.com/googleapis/java-storage/issues/2605)) ([a532ee4](https://github.com/googleapis/java-storage/commit/a532ee49e2ff5972ea8a2aabbab2dcf6fe0df774)) + ## [2.40.1](https://github.com/googleapis/java-storage/compare/v2.40.0...v2.40.1) (2024-06-26) diff --git a/README.md b/README.md index dada3cb6c6..fa7eb5ba54 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud libraries-bom - 26.43.0 + 26.44.0 pom import @@ -35,6 +35,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud google-cloud-storage-control + ``` @@ -46,12 +47,12 @@ If you are using Maven without the BOM, add this to your dependencies: com.google.cloud google-cloud-storage - 2.40.1 + 2.41.0 com.google.cloud google-cloud-storage-control - 2.40.2-SNAPSHOT + 2.41.0 ``` @@ -59,20 +60,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.43.0') +implementation platform('com.google.cloud:libraries-bom:26.44.0') implementation 'com.google.cloud:google-cloud-storage' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-storage:2.40.1' +implementation 'com.google.cloud:google-cloud-storage:2.41.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.40.1" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.41.0" ``` @@ -115,13 +116,14 @@ The [Storage Control API](https://cloud.google.com/storage/docs/reference/rpc/) The Storage Control API creates one space to perform metadata-specific, control plane, and long-running operations apart from the Storage API. Separating these operations from the Storage API improves API standardization and lets you run faster releases. If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + ```xml com.google.cloud libraries-bom - 26.37.0 + 26.44.0 pom import @@ -130,30 +132,33 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: - com.google.cloud - google-cloud-storage-control - + com.google.cloud + google-cloud-storage-control + + ``` If you are using Maven without the BOM, add this to your dependencies: - + + + ```xml com.google.cloud google-cloud-storage-control - 2.40.2-SNAPSHOT + 2.41.0 ``` If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:2.40.1') +implementation platform('com.google.cloud:libraries-bom:26.44.0') implementation 'com.google.cloud:google-cloud-storage-control' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-storage-control:2.40.2-SNAPSHOT' +implementation 'com.google.cloud:google-cloud-storage-control:2.41.0' ``` #### Creating an authorized service object @@ -277,7 +282,6 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-storage/tree/ | Sample | Source Code | Try it | | --------------------------- | --------------------------------- | ------ | -| Native Image Storage Sample | [source code](https://github.com/googleapis/java-storage/blob/main/samples/native-image-sample/src/main/java/com/example/storage/NativeImageStorageSample.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-storage&page=editor&open_in_editor=samples/native-image-sample/src/main/java/com/example/storage/NativeImageStorageSample.java) | | Configure Retries | [source code](https://github.com/googleapis/java-storage/blob/main/samples/snippets/src/main/java/com/example/storage/ConfigureRetries.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-storage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/storage/ConfigureRetries.java) | | Generate Signed Post Policy V4 | [source code](https://github.com/googleapis/java-storage/blob/main/samples/snippets/src/main/java/com/example/storage/GenerateSignedPostPolicyV4.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-storage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/storage/GenerateSignedPostPolicyV4.java) | | Get Service Account | [source code](https://github.com/googleapis/java-storage/blob/main/samples/snippets/src/main/java/com/example/storage/GetServiceAccount.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-storage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/storage/GetServiceAccount.java) | @@ -414,7 +418,7 @@ To get help, follow the instructions in the [shared Troubleshooting document][tr ## Transport -Cloud Storage uses gRPC for the transport layer. +Cloud Storage uses HTTP/JSON for the transport layer. ## Supported Java Versions @@ -508,7 +512,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-storage/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-storage.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.40.1 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.41.0 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/gapic-google-cloud-storage-v2/pom.xml b/gapic-google-cloud-storage-v2/pom.xml index 76bc3173df..3774a7ef3b 100644 --- a/gapic-google-cloud-storage-v2/pom.xml +++ b/gapic-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc gapic-google-cloud-storage-v2 - 2.40.2-alpha-SNAPSHOT + 2.41.1-alpha-SNAPSHOT gapic-google-cloud-storage-v2 GRPC library for gapic-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT diff --git a/gapic-google-cloud-storage-v2/src/main/java/com/google/storage/v2/stub/StorageStubSettings.java b/gapic-google-cloud-storage-v2/src/main/java/com/google/storage/v2/stub/StorageStubSettings.java index 1b42b3c7e2..3ced26a1a9 100644 --- a/gapic-google-cloud-storage-v2/src/main/java/com/google/storage/v2/stub/StorageStubSettings.java +++ b/gapic-google-cloud-storage-v2/src/main/java/com/google/storage/v2/stub/StorageStubSettings.java @@ -23,6 +23,7 @@ import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; +import com.google.api.core.ObsoleteApi; import com.google.api.gax.core.GaxProperties; import com.google.api.gax.core.GoogleCredentialsProvider; import com.google.api.gax.core.InstantiatingExecutorProvider; @@ -637,6 +638,7 @@ public static InstantiatingExecutorProvider.Builder defaultExecutorProviderBuild } /** Returns the default service endpoint. */ + @ObsoleteApi("Use getEndpoint() instead") public static String getDefaultEndpoint() { return "storage.googleapis.com:443"; } diff --git a/generation_config.yaml b/generation_config.yaml index fc8de99ad8..06d8418e92 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,6 +1,6 @@ -gapic_generator_version: 2.42.0 -googleapis_commitish: 19577edb4d439db98d2fb1f6f48f2e1b29fba099 -libraries_bom_version: 26.43.0 +gapic_generator_version: 2.43.0 +googleapis_commitish: 195c051374369ac270384e622f05b2b961dcacd5 +libraries_bom_version: 26.44.0 libraries: - api_shortname: storage name_pretty: Cloud Storage @@ -20,6 +20,7 @@ libraries: extra_versioned_modules: gapic-google-cloud-storage-v2 excluded_poms: google-cloud-storage-bom,google-cloud-storage recommended_package: com.google.cloud.storage + transport: rest GAPICs: - proto_path: google/storage/v2 - proto_path: google/storage/control/v2 diff --git a/google-cloud-storage-bom/pom.xml b/google-cloud-storage-bom/pom.xml index 8430a543b0..898fd19910 100644 --- a/google-cloud-storage-bom/pom.xml +++ b/google-cloud-storage-bom/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud google-cloud-storage-bom - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT pom com.google.cloud sdk-platform-java-config - 3.32.0 + 3.33.0 @@ -69,37 +69,37 @@ com.google.cloud google-cloud-storage - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT com.google.api.grpc gapic-google-cloud-storage-v2 - 2.40.2-alpha-SNAPSHOT + 2.41.1-alpha-SNAPSHOT com.google.api.grpc grpc-google-cloud-storage-v2 - 2.40.2-alpha-SNAPSHOT + 2.41.1-alpha-SNAPSHOT com.google.api.grpc proto-google-cloud-storage-v2 - 2.40.2-alpha-SNAPSHOT + 2.41.1-alpha-SNAPSHOT com.google.cloud google-cloud-storage-control - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT com.google.api.grpc grpc-google-cloud-storage-control-v2 - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT com.google.api.grpc proto-google-cloud-storage-control-v2 - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT diff --git a/google-cloud-storage-control/pom.xml b/google-cloud-storage-control/pom.xml index 799e37f0ed..76e59f8138 100644 --- a/google-cloud-storage-control/pom.xml +++ b/google-cloud-storage-control/pom.xml @@ -5,13 +5,13 @@ 4.0.0 com.google.cloud google-cloud-storage-control - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT google-cloud-storage-control GRPC library for google-cloud-storage-control com.google.cloud google-cloud-storage-parent - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT diff --git a/google-cloud-storage-control/src/main/java/com/google/storage/control/v2/stub/StorageControlStubSettings.java b/google-cloud-storage-control/src/main/java/com/google/storage/control/v2/stub/StorageControlStubSettings.java index eb717d3824..b9231bb537 100644 --- a/google-cloud-storage-control/src/main/java/com/google/storage/control/v2/stub/StorageControlStubSettings.java +++ b/google-cloud-storage-control/src/main/java/com/google/storage/control/v2/stub/StorageControlStubSettings.java @@ -21,6 +21,7 @@ import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; +import com.google.api.core.ObsoleteApi; import com.google.api.gax.core.GaxProperties; import com.google.api.gax.core.GoogleCredentialsProvider; import com.google.api.gax.core.InstantiatingExecutorProvider; @@ -334,6 +335,7 @@ public static InstantiatingExecutorProvider.Builder defaultExecutorProviderBuild } /** Returns the default service endpoint. */ + @ObsoleteApi("Use getEndpoint() instead") public static String getDefaultEndpoint() { return "storage.googleapis.com:443"; } diff --git a/google-cloud-storage-control/src/main/resources/META-INF/native-image/com.google.storage.control.v2/reflect-config.json b/google-cloud-storage-control/src/main/resources/META-INF/native-image/com.google.storage.control.v2/reflect-config.json index 36399f5086..c10085e0b1 100644 --- a/google-cloud-storage-control/src/main/resources/META-INF/native-image/com.google.storage.control.v2/reflect-config.json +++ b/google-cloud-storage-control/src/main/resources/META-INF/native-image/com.google.storage.control.v2/reflect-config.json @@ -440,6 +440,24 @@ "allDeclaredClasses": true, "allPublicClasses": true }, + { + "name": "com.google.api.TypeReference", + "queryAllDeclaredConstructors": true, + "queryAllPublicConstructors": true, + "queryAllDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "com.google.api.TypeReference$Builder", + "queryAllDeclaredConstructors": true, + "queryAllPublicConstructors": true, + "queryAllDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, { "name": "com.google.longrunning.CancelOperationRequest", "queryAllDeclaredConstructors": true, diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index b6bc155e3d..8a44a6474d 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,12 +12,12 @@ com.google.cloud google-cloud-storage-parent - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT google-cloud-storage - 1.113.0 - 5.10.3 + 1.114.0 + 5.11.0 @@ -124,6 +124,53 @@ com.google.api.grpc gapic-google-cloud-storage-v2 + + + io.opentelemetry + opentelemetry-sdk + + + io.grpc + grpc-opentelemetry + + + io.opentelemetry + opentelemetry-api + + + + io.opentelemetry + opentelemetry-sdk-metrics + + + io.opentelemetry + opentelemetry-sdk-common + + + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure-spi + + + + io.opentelemetry.semconv + opentelemetry-semconv + + + + com.google.cloud.opentelemetry + exporter-metrics + + + + io.opentelemetry.contrib + opentelemetry-gcp-resources + + + + org.checkerframework + checker-qual + @@ -159,10 +206,7 @@ grpc-googleapis runtime - - org.checkerframework - checker-qual - + org.junit.vintage:junit-vintage-engine + + + io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi + io.opentelemetry.semconv:opentelemetry-semconv diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultStorageRetryStrategy.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultStorageRetryStrategy.java index cf85b7400d..b9a8df0b52 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultStorageRetryStrategy.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultStorageRetryStrategy.java @@ -26,6 +26,7 @@ import com.google.gson.stream.MalformedJsonException; import java.io.IOException; import java.net.SocketException; +import java.net.UnknownHostException; import java.util.Set; import javax.net.ssl.SSLException; @@ -114,6 +115,8 @@ private RetryResult shouldRetryIOException(IOException ioException) { SocketException se = (SocketException) cause; return shouldRetryIOException(se); } + } else if (ioException instanceof UnknownHostException && idempotent) { + return RetryResult.RETRY; } if (BaseServiceException.isRetryable(idempotent, ioException)) { return RetryResult.RETRY; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedFinalizeOnCloseResumableWritableByteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedFinalizeOnCloseResumableWritableByteChannel.java index 984c7bfd8f..e85363b227 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedFinalizeOnCloseResumableWritableByteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedFinalizeOnCloseResumableWritableByteChannel.java @@ -17,6 +17,7 @@ package com.google.cloud.storage; import static com.google.cloud.storage.GrpcUtils.contextWithBucketName; +import static com.google.cloud.storage.Utils.nullSafeList; import com.google.api.core.SettableApiFuture; import com.google.api.gax.grpc.GrpcCallContext; @@ -26,7 +27,6 @@ import com.google.cloud.storage.ChunkSegmenter.ChunkSegment; import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown; import com.google.cloud.storage.UnbufferedWritableByteChannelSession.UnbufferedWritableByteChannel; -import com.google.common.collect.ImmutableList; import com.google.protobuf.ByteString; import com.google.storage.v2.ChecksummedData; import com.google.storage.v2.ObjectChecksums; @@ -230,7 +230,7 @@ public void onError(Throwable t) { tmp.getCode(), tmp.getMessage(), tmp.getReason(), - ImmutableList.of(lastWrittenRequest), + nullSafeList(lastWrittenRequest), null, context, t); @@ -251,7 +251,7 @@ public void onCompleted() { 0, "onComplete without preceding onNext, unable to determine success.", "invalid", - ImmutableList.of(lastWrittenRequest), + nullSafeList(lastWrittenRequest), null, context, null)); @@ -263,11 +263,11 @@ public void onCompleted() { } else if (finalSize < totalSentBytes) { clientDetectedError( ResumableSessionFailureScenario.SCENARIO_4_1.toStorageException( - ImmutableList.of(lastWrittenRequest), last, context, null)); + nullSafeList(lastWrittenRequest), last, context, null)); } else { clientDetectedError( ResumableSessionFailureScenario.SCENARIO_4_2.toStorageException( - ImmutableList.of(lastWrittenRequest), last, context, null)); + nullSafeList(lastWrittenRequest), last, context, null)); } } else if (!finalizing || last.hasPersistedSize()) { // unexpected incremental response clientDetectedError( @@ -275,14 +275,14 @@ public void onCompleted() { 0, "Unexpected incremental response for finalizing request.", "invalid", - ImmutableList.of(lastWrittenRequest), + nullSafeList(lastWrittenRequest), last, context, null)); } else { clientDetectedError( ResumableSessionFailureScenario.SCENARIO_0.toStorageException( - ImmutableList.of(lastWrittenRequest), last, context, null)); + nullSafeList(lastWrittenRequest), last, context, null)); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java index 92081e03e0..64376cc107 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java @@ -116,6 +116,9 @@ public final class GrpcStorageOptions extends StorageOptions private final GrpcRetryAlgorithmManager retryAlgorithmManager; private final Duration terminationAwaitDuration; private final boolean attemptDirectPath; + private final boolean enableGrpcClientMetrics; + + private final boolean grpcClientMetricsManuallyEnabled; private final GrpcInterceptorProvider grpcInterceptorProvider; private final BlobWriteSessionConfig blobWriteSessionConfig; @@ -129,6 +132,8 @@ private GrpcStorageOptions(Builder builder, GrpcStorageDefaults serviceDefaults) MoreObjects.firstNonNull( builder.terminationAwaitDuration, serviceDefaults.getTerminationAwaitDuration()); this.attemptDirectPath = builder.attemptDirectPath; + this.enableGrpcClientMetrics = builder.enableGrpcClientMetrics; + this.grpcClientMetricsManuallyEnabled = builder.grpcMetricsManuallyEnabled; this.grpcInterceptorProvider = builder.grpcInterceptorProvider; this.blobWriteSessionConfig = builder.blobWriteSessionConfig; } @@ -287,6 +292,16 @@ private Tuple> resolveSettingsAndOpts() throw if (scheme.equals("http")) { channelProviderBuilder.setChannelConfigurator(ManagedChannelBuilder::usePlaintext); } + + if (enableGrpcClientMetrics) { + OpenTelemetryBootstrappingUtils.enableGrpcMetrics( + channelProviderBuilder, + endpoint, + this.getProjectId(), + this.getUniverseDomain(), + !grpcClientMetricsManuallyEnabled); + } + builder.setTransportChannelProvider(channelProviderBuilder.build()); RetrySettings baseRetrySettings = getRetrySettings(); RetrySettings readRetrySettings = @@ -350,6 +365,7 @@ public int hashCode() { retryAlgorithmManager, terminationAwaitDuration, attemptDirectPath, + enableGrpcClientMetrics, grpcInterceptorProvider, blobWriteSessionConfig, baseHashCode()); @@ -365,6 +381,7 @@ public boolean equals(Object o) { } GrpcStorageOptions that = (GrpcStorageOptions) o; return attemptDirectPath == that.attemptDirectPath + && enableGrpcClientMetrics == that.enableGrpcClientMetrics && Objects.equals(retryAlgorithmManager, that.retryAlgorithmManager) && Objects.equals(terminationAwaitDuration, that.terminationAwaitDuration) && Objects.equals(grpcInterceptorProvider, that.grpcInterceptorProvider) @@ -408,11 +425,15 @@ public static final class Builder extends StorageOptions.Builder { private StorageRetryStrategy storageRetryStrategy; private Duration terminationAwaitDuration; private boolean attemptDirectPath = GrpcStorageDefaults.INSTANCE.isAttemptDirectPath(); + private boolean enableGrpcClientMetrics = + GrpcStorageDefaults.INSTANCE.isEnableGrpcClientMetrics(); private GrpcInterceptorProvider grpcInterceptorProvider = GrpcStorageDefaults.INSTANCE.grpcInterceptorProvider(); private BlobWriteSessionConfig blobWriteSessionConfig = GrpcStorageDefaults.INSTANCE.getDefaultStorageWriterConfig(); + private boolean grpcMetricsManuallyEnabled = false; + Builder() {} Builder(StorageOptions options) { @@ -421,6 +442,7 @@ public static final class Builder extends StorageOptions.Builder { this.storageRetryStrategy = gso.getRetryAlgorithmManager().retryStrategy; this.terminationAwaitDuration = gso.getTerminationAwaitDuration(); this.attemptDirectPath = gso.attemptDirectPath; + this.enableGrpcClientMetrics = gso.enableGrpcClientMetrics; this.grpcInterceptorProvider = gso.grpcInterceptorProvider; this.blobWriteSessionConfig = gso.blobWriteSessionConfig; } @@ -454,6 +476,21 @@ public GrpcStorageOptions.Builder setAttemptDirectPath(boolean attemptDirectPath this.attemptDirectPath = attemptDirectPath; return this; } + /** + * Option for whether this client should emit internal gRPC client internal metrics to Cloud + * Monitoring. To disable metric reporting, set this to false. True by default. Emitting metrics + * is free and requires minimal CPU and memory. + * + * @since 2.41.0 This new api is in preview and is subject to breaking changes. + */ + @BetaApi + public GrpcStorageOptions.Builder setEnableGrpcClientMetrics(boolean enableGrpcClientMetrics) { + this.enableGrpcClientMetrics = enableGrpcClientMetrics; + if (enableGrpcClientMetrics) { + grpcMetricsManuallyEnabled = true; + } + return this; + } /** @since 2.14.0 This new api is in preview and is subject to breaking changes. */ @BetaApi @@ -660,6 +697,12 @@ public boolean isAttemptDirectPath() { return false; } + /** @since 2.41.0 This new api is in preview and is subject to breaking changes. */ + @BetaApi + public boolean isEnableGrpcClientMetrics() { + return true; + } + /** @since 2.22.3 This new api is in preview and is subject to breaking changes. */ @BetaApi public GrpcInterceptorProvider grpcInterceptorProvider() { diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java index 7f84db77d0..274441436a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java @@ -83,6 +83,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; @@ -1082,6 +1083,11 @@ private static Function, List> toListOf(Function f // various data level methods in the apiary model are hostile to ImmutableList, as it does not // provide a public default no args constructor. Instead, apiary uses ArrayList for all internal // representations of JSON Arrays. - return l -> l.stream().map(f).collect(Collectors.toList()); + return l -> { + if (l == null) { + return ImmutableList.of(); + } + return l.stream().filter(Objects::nonNull).map(f).collect(Collectors.toList()); + }; } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java new file mode 100644 index 0000000000..fcf1440b49 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java @@ -0,0 +1,383 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import com.google.api.core.ApiFunction; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.api.gax.rpc.PermissionDeniedException; +import com.google.api.gax.rpc.UnavailableException; +import com.google.cloud.opentelemetry.metric.GoogleCloudMetricExporter; +import com.google.cloud.opentelemetry.metric.MetricConfiguration; +import com.google.cloud.opentelemetry.metric.MonitoredResourceDescription; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import io.grpc.ManagedChannelBuilder; +import io.grpc.opentelemetry.GrpcOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.internal.StringUtils; +import io.opentelemetry.contrib.gcp.resource.GCPResourceProvider; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.common.export.MemoryMode; +import io.opentelemetry.sdk.metrics.Aggregation; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.View; +import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.export.DefaultAggregationSelector; +import io.opentelemetry.sdk.metrics.export.MetricExporter; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.resources.Resource; +import java.math.BigDecimal; +import java.math.MathContext; +import java.net.NoRouteToHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +final class OpenTelemetryBootstrappingUtils { + private static final Collection METRICS_TO_ENABLE = + ImmutableList.of( + "grpc.lb.wrr.rr_fallback", + "grpc.lb.wrr.endpoint_weight_not_yet_usable", + "grpc.lb.wrr.endpoint_weight_stale", + "grpc.lb.wrr.endpoint_weights", + "grpc.lb.rls.cache_entries", + "grpc.lb.rls.cache_size", + "grpc.lb.rls.default_target_picks", + "grpc.lb.rls.target_picks", + "grpc.lb.rls.failed_picks", + "grpc.xds_client.connected", + "grpc.xds_client.server_failure", + "grpc.xds_client.resource_updates_valid", + "grpc.xds_client.resource_updates_invalid", + "grpc.xds_client.resources"); + + private static final Collection METRICS_ENABLED_BY_DEFAULT = + ImmutableList.of( + "grpc.client.attempt.sent_total_compressed_message_size", + "grpc.client.attempt.rcvd_total_compressed_message_size", + "grpc.client.attempt.started", + "grpc.client.attempt.duration", + "grpc.client.call.duration"); + + static final Logger log = Logger.getLogger(OpenTelemetryBootstrappingUtils.class.getName()); + + static void enableGrpcMetrics( + InstantiatingGrpcChannelProvider.Builder channelProviderBuilder, + String endpoint, + String projectId, + String universeDomain, + boolean shouldSuppressExceptions) { + String metricServiceEndpoint = getCloudMonitoringEndpoint(endpoint, universeDomain); + SdkMeterProvider provider = + createMeterProvider(metricServiceEndpoint, projectId, shouldSuppressExceptions); + + OpenTelemetrySdk openTelemetrySdk = + OpenTelemetrySdk.builder().setMeterProvider(provider).build(); + GrpcOpenTelemetry grpcOpenTelemetry = + GrpcOpenTelemetry.newBuilder() + .sdk(openTelemetrySdk) + .addOptionalLabel("grpc.lb.locality") + .enableMetrics(METRICS_TO_ENABLE) + .build(); + ApiFunction channelConfigurator = + channelProviderBuilder.getChannelConfigurator(); + channelProviderBuilder.setChannelConfigurator( + b -> { + grpcOpenTelemetry.configureChannelBuilder(b); + if (channelConfigurator != null) { + return channelConfigurator.apply(b); + } + return b; + }); + } + + @VisibleForTesting + static String getCloudMonitoringEndpoint(String endpoint, String universeDomain) { + String metricServiceEndpoint = "monitoring.googleapis.com"; + + // use contains instead of equals because endpoint has a port in it + if (universeDomain != null && endpoint.contains("storage." + universeDomain)) { + metricServiceEndpoint = "monitoring." + universeDomain; + } else if (!endpoint.contains("storage.googleapis.com")) { + String canonicalEndpoint = "storage.googleapis.com"; + String privateEndpoint = "private.googleapis.com"; + String restrictedEndpoint = "restricted.googleapis.com"; + if (universeDomain != null) { + canonicalEndpoint = "storage." + universeDomain; + privateEndpoint = "private." + universeDomain; + restrictedEndpoint = "restricted." + universeDomain; + } + String match = + ImmutableList.of(canonicalEndpoint, privateEndpoint, restrictedEndpoint).stream() + .filter(s -> endpoint.contains(s) || endpoint.contains("google-c2p:///" + s)) + .collect(Collectors.joining()); + if (!StringUtils.isNullOrEmpty(match)) { + metricServiceEndpoint = match; + } + } + return metricServiceEndpoint + ":" + endpoint.split(":")[1]; + } + + @VisibleForTesting + static SdkMeterProvider createMeterProvider( + String metricServiceEndpoint, String projectId, boolean shouldSuppressExceptions) { + GCPResourceProvider resourceProvider = new GCPResourceProvider(); + Attributes detectedAttributes = resourceProvider.getAttributes(); + + String detectedProjectId = detectedAttributes.get(AttributeKey.stringKey("cloud.account.id")); + String projectIdToUse = detectedProjectId == null ? projectId : detectedProjectId; + + if (!projectIdToUse.equals(projectId)) { + log.warning( + "The Project ID configured for metrics is " + + projectIdToUse + + ", but the Project ID of the storage client is " + + projectId + + ". Make sure that the service account in use has the required metric writing role " + + "(roles/monitoring.metricWriter) in the project " + + projectIdToUse + + ", or metrics will not be written."); + } + + MonitoredResourceDescription monitoredResourceDescription = + new MonitoredResourceDescription( + "storage.googleapis.com/Client", + ImmutableSet.of( + "project_id", "location", "cloud_platform", "host_id", "instance_id", "api")); + + MetricExporter cloudMonitoringExporter = + GoogleCloudMetricExporter.createWithConfiguration( + MetricConfiguration.builder() + .setMonitoredResourceDescription(monitoredResourceDescription) + .setInstrumentationLibraryLabelsEnabled(false) + .setMetricServiceEndpoint(metricServiceEndpoint) + .setPrefix("storage.googleapis.com/client") + .setUseServiceTimeSeries(true) + .setProjectId(projectIdToUse) + .build()); + + SdkMeterProviderBuilder providerBuilder = SdkMeterProvider.builder(); + + // This replaces the dots with slashes in each metric, which is the format needed for this + // monitored resource + for (String metric : + ImmutableList.copyOf(Iterables.concat(METRICS_TO_ENABLE, METRICS_ENABLED_BY_DEFAULT))) { + providerBuilder.registerView( + InstrumentSelector.builder().setName(metric).build(), + View.builder().setName(metric.replace(".", "/")).build()); + } + MetricExporter exporter = + shouldSuppressExceptions + ? new PermissionDeniedSingleReportMetricsExporter(cloudMonitoringExporter) + : cloudMonitoringExporter; + providerBuilder + .registerMetricReader( + PeriodicMetricReader.builder(exporter) + .setInterval(java.time.Duration.ofSeconds(60)) + .build()) + .setResource( + Resource.create( + Attributes.builder() + .put("gcp.resource_type", "storage.googleapis.com/Client") + .put("location", detectedAttributes.get(AttributeKey.stringKey("cloud.region"))) + .put("project_id", projectIdToUse) + .put( + "cloud_platform", + detectedAttributes.get(AttributeKey.stringKey("cloud.platform"))) + .put("host_id", detectedAttributes.get(AttributeKey.stringKey("host.id"))) + .put("instance_id", UUID.randomUUID().toString()) + .put("api", "grpc") + .build())); + + addHistogramView( + providerBuilder, latencyHistogramBoundaries(), "grpc/client/attempt/duration", "s"); + addHistogramView( + providerBuilder, + sizeHistogramBoundaries(), + "grpc/client/attempt/rcvd_total_compressed_message_size", + "By"); + addHistogramView( + providerBuilder, + sizeHistogramBoundaries(), + "grpc/client/attempt/sent_total_compressed_message_size", + "By"); + + return providerBuilder.build(); + } + + private static void addHistogramView( + SdkMeterProviderBuilder provider, List boundaries, String name, String unit) { + InstrumentSelector instrumentSelector = + InstrumentSelector.builder() + .setType(InstrumentType.HISTOGRAM) + .setUnit(unit) + .setName(name) + .setMeterName("grpc-java") + .setMeterSchemaUrl("") + .build(); + View view = + View.builder() + .setName(name) + .setDescription( + "A view of " + + name + + " with histogram boundaries more appropriate for Google Cloud Storage RPCs") + .setAggregation(Aggregation.explicitBucketHistogram(boundaries)) + .build(); + provider.registerView(instrumentSelector, view); + } + + private static List latencyHistogramBoundaries() { + List boundaries = new ArrayList<>(); + BigDecimal boundary = new BigDecimal(0, MathContext.UNLIMITED); + BigDecimal increment = new BigDecimal("0.002", MathContext.UNLIMITED); // 2ms + + // 2ms buckets for the first 100ms, so we can have higher resolution for uploads and downloads + // in the 100 KiB range + for (int i = 0; i != 50; i++) { + boundaries.add(boundary.doubleValue()); + boundary = boundary.add(increment); + } + + // For the remaining buckets do 10 10ms, 10 20ms, and so on, up until 5 minutes + increment = new BigDecimal("0.01", MathContext.UNLIMITED); // 10 ms + for (int i = 0; i != 150 && boundary.compareTo(new BigDecimal(300)) < 1; i++) { + boundaries.add(boundary.doubleValue()); + if (i != 0 && i % 10 == 0) { + increment = increment.multiply(new BigDecimal(2)); + } + boundary = boundary.add(increment); + } + + return boundaries; + } + + private static List sizeHistogramBoundaries() { + long kb = 1024; + long mb = 1024 * kb; + long gb = 1024 * mb; + + List boundaries = new ArrayList<>(); + long boundary = 0; + long increment = 128 * kb; + + // 128 KiB increments up to 4MiB, then exponential growth + while (boundaries.size() < 200 && boundary <= 16 * gb) { + boundaries.add((double) boundary); + boundary += increment; + if (boundary >= 4 * mb) { + increment *= 2; + } + } + return boundaries; + } + + private static final class PermissionDeniedSingleReportMetricsExporter implements MetricExporter { + private final MetricExporter delegate; + private final AtomicBoolean seenPermissionDenied = new AtomicBoolean(false); + private final AtomicBoolean seenNoRouteToHost = new AtomicBoolean(false); + + private PermissionDeniedSingleReportMetricsExporter(MetricExporter delegate) { + this.delegate = delegate; + } + + @Override + public CompletableResultCode export(Collection metrics) { + if (seenPermissionDenied.get() && seenNoRouteToHost.get()) { + return CompletableResultCode.ofFailure(); + } + + try { + return delegate.export(metrics); + } catch (PermissionDeniedException e) { + if (!seenPermissionDenied.get()) { + seenPermissionDenied.set(true); + throw e; + } + return CompletableResultCode.ofFailure(); + } catch (UnavailableException e) { + if (seenPermissionDenied.get() + && !seenNoRouteToHost.get() + && ultimateCause(e, NoRouteToHostException.class)) { + seenNoRouteToHost.set(true); + throw e; + } + return CompletableResultCode.ofFailure(); + } + } + + @Override + public Aggregation getDefaultAggregation(InstrumentType instrumentType) { + return delegate.getDefaultAggregation(instrumentType); + } + + @Override + public MemoryMode getMemoryMode() { + return delegate.getMemoryMode(); + } + + @Override + public CompletableResultCode flush() { + return delegate.flush(); + } + + @Override + public CompletableResultCode shutdown() { + return delegate.shutdown(); + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public AggregationTemporality getAggregationTemporality(InstrumentType instrumentType) { + return delegate.getAggregationTemporality(instrumentType); + } + + @Override + public DefaultAggregationSelector with(InstrumentType instrumentType, Aggregation aggregation) { + return delegate.with(instrumentType, aggregation); + } + + private static boolean ultimateCause(Throwable t, Class c) { + if (t == null) { + return false; + } + + Throwable cause = t.getCause(); + if (cause != null && c.isAssignableFrom(cause.getClass())) { + return true; + } else { + return ultimateCause(cause, c); + } + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSessionFailureScenario.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSessionFailureScenario.java index 88dbaf7bdc..aab5263397 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSessionFailureScenario.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSessionFailureScenario.java @@ -169,6 +169,9 @@ static StorageException toStorageException( Map> extraHeaders = context.getExtraHeaders(); recordHeadersTo(extraHeaders, PREFIX_O, sb); int length = reqs.size(); + if (length == 0) { + sb.append("\n").append(PREFIX_O).append("[]"); + } for (int i = 0; i < length; i++) { if (i == 0) { sb.append("\n").append(PREFIX_O).append("["); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index fca1d1294d..89624d9b0a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -45,11 +45,13 @@ import com.google.cloud.storage.UnifiedOpts.HmacKeySourceOpt; import com.google.cloud.storage.UnifiedOpts.HmacKeyTargetOpt; import com.google.cloud.storage.UnifiedOpts.NamedField; +import com.google.cloud.storage.UnifiedOpts.NestedNamedField; import com.google.cloud.storage.UnifiedOpts.ObjectListOpt; import com.google.cloud.storage.UnifiedOpts.ObjectSourceOpt; import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt; import com.google.cloud.storage.UnifiedOpts.Opts; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Streams; @@ -62,9 +64,11 @@ import java.net.URLConnection; import java.nio.file.Path; import java.security.Key; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -73,6 +77,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; /** * An interface for Google Cloud Storage. @@ -106,80 +111,107 @@ String getEntry() { enum BucketField implements FieldSelector, NamedField { @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - ID("id", "bucket_id"), + ID("id", "bucket_id", String.class), @TransportCompatibility(Transport.HTTP) - SELF_LINK("selfLink"), + SELF_LINK("selfLink", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - NAME("name"), + NAME("name", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - TIME_CREATED("timeCreated", "create_time"), + TIME_CREATED("timeCreated", "create_time", com.google.api.client.util.DateTime.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - METAGENERATION("metageneration"), + METAGENERATION("metageneration", Long.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - ACL("acl"), + ACL("acl", ArrayList.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - DEFAULT_OBJECT_ACL("defaultObjectAcl", "default_object_acl"), + DEFAULT_OBJECT_ACL("defaultObjectAcl", "default_object_acl", ArrayList.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - OWNER("owner"), + OWNER("owner", com.google.api.services.storage.model.Bucket.Owner.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - LABELS("labels"), + LABELS("labels", HashMap.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - LOCATION("location"), + LOCATION("location", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - LOCATION_TYPE("locationType", "location_type"), + LOCATION_TYPE("locationType", "location_type", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - WEBSITE("website"), + WEBSITE("website", com.google.api.services.storage.model.Bucket.Website.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - VERSIONING("versioning"), + VERSIONING("versioning", com.google.api.services.storage.model.Bucket.Versioning.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CORS("cors"), + CORS("cors", ArrayList.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - LIFECYCLE("lifecycle"), + LIFECYCLE("lifecycle", com.google.api.services.storage.model.Bucket.Lifecycle.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - STORAGE_CLASS("storageClass", "storage_class"), + STORAGE_CLASS("storageClass", "storage_class", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - ETAG("etag"), + ETAG("etag", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - ENCRYPTION("encryption"), + ENCRYPTION("encryption", com.google.api.services.storage.model.Bucket.Encryption.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - BILLING("billing"), + BILLING("billing", com.google.api.services.storage.model.Bucket.Billing.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - DEFAULT_EVENT_BASED_HOLD("defaultEventBasedHold", "default_event_based_hold"), + DEFAULT_EVENT_BASED_HOLD("defaultEventBasedHold", "default_event_based_hold", Boolean.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - RETENTION_POLICY("retentionPolicy", "retention_policy"), + RETENTION_POLICY( + "retentionPolicy", + "retention_policy", + com.google.api.services.storage.model.Bucket.RetentionPolicy.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - IAMCONFIGURATION("iamConfiguration", "iam_config"), + IAMCONFIGURATION( + "iamConfiguration", + "iam_config", + com.google.api.services.storage.model.Bucket.IamConfiguration.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - LOGGING("logging"), + LOGGING("logging", com.google.api.services.storage.model.Bucket.Logging.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - UPDATED("updated", "update_time"), + UPDATED("updated", "update_time", com.google.api.client.util.DateTime.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - RPO("rpo"), + RPO("rpo", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CUSTOM_PLACEMENT_CONFIG("customPlacementConfig", "custom_placement_config"), + CUSTOM_PLACEMENT_CONFIG( + "customPlacementConfig", + "custom_placement_config", + com.google.api.services.storage.model.Bucket.CustomPlacementConfig.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - AUTOCLASS("autoclass"), + AUTOCLASS("autoclass", com.google.api.services.storage.model.Bucket.Autoclass.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - HIERARCHICAL_NAMESPACE("hierarchicalNamespace", "hierarchical_namespace"), + HIERARCHICAL_NAMESPACE( + "hierarchicalNamespace", + "hierarchical_namespace", + com.google.api.services.storage.model.Bucket.HierarchicalNamespace.class), @TransportCompatibility({Transport.HTTP}) - OBJECT_RETENTION("objectRetention"), + OBJECT_RETENTION( + "objectRetention", com.google.api.services.storage.model.Bucket.ObjectRetention.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - SOFT_DELETE_POLICY("softDeletePolicy", "soft_delete_policy"); + SOFT_DELETE_POLICY( + "softDeletePolicy", + "soft_delete_policy", + com.google.api.services.storage.model.Bucket.SoftDeletePolicy.class); static final List REQUIRED_FIELDS = ImmutableList.of(NAME); + private static final Map JSON_FIELD_NAME_INDEX; + + static { + ImmutableMap.Builder tmp = ImmutableMap.builder(); + for (BucketField field : values()) { + tmp.put(field.selector, field); + } + JSON_FIELD_NAME_INDEX = Utils.mapBuild(tmp); + } private final String selector; private final String grpcFieldName; + private final Class jsonClass; - BucketField(String selector) { - this(selector, selector); + BucketField(String selector, Class jsonClass) { + this(selector, selector, jsonClass); } - BucketField(String selector, String grpcFieldName) { + BucketField(String selector, String grpcFieldName, Class jsonClass) { this.selector = selector; this.grpcFieldName = grpcFieldName; + this.jsonClass = jsonClass; } @Override @@ -196,96 +228,129 @@ public String getApiaryName() { public String getGrpcName() { return grpcFieldName; } + + Class getJsonClass() { + return jsonClass; + } + + @Nullable + static BucketField lookup(NamedField nf) { + NamedField lookup = nf; + if (nf instanceof NestedNamedField) { + NestedNamedField nested = (NestedNamedField) nf; + lookup = nested.getParent(); + } + return JSON_FIELD_NAME_INDEX.get(lookup.getApiaryName()); + } } enum BlobField implements FieldSelector, NamedField { @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - ACL("acl"), + ACL("acl", com.google.api.services.storage.model.ObjectAccessControl.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - BUCKET("bucket"), + BUCKET("bucket", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CACHE_CONTROL("cacheControl", "cache_control"), + CACHE_CONTROL("cacheControl", "cache_control", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - COMPONENT_COUNT("componentCount", "component_count"), + COMPONENT_COUNT("componentCount", "component_count", Integer.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CONTENT_DISPOSITION("contentDisposition", "content_disposition"), + CONTENT_DISPOSITION("contentDisposition", "content_disposition", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CONTENT_ENCODING("contentEncoding", "content_encoding"), + CONTENT_ENCODING("contentEncoding", "content_encoding", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CONTENT_LANGUAGE("contentLanguage", "content_language"), + CONTENT_LANGUAGE("contentLanguage", "content_language", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CONTENT_TYPE("contentType", "content_type"), + CONTENT_TYPE("contentType", "content_type", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CRC32C("crc32c", "checksums.crc32c"), + CRC32C("crc32c", "checksums.crc32c", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - ETAG("etag"), + ETAG("etag", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - GENERATION("generation"), + GENERATION("generation", Long.class), @TransportCompatibility(Transport.HTTP) - ID("id"), + ID("id", String.class), /** {@code kind} is not exposed in {@link BlobInfo} or {@link Blob} no need to select it */ @Deprecated @TransportCompatibility(Transport.HTTP) - KIND("kind"), + KIND("kind", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - MD5HASH("md5Hash", "checksums.md5_hash"), + MD5HASH("md5Hash", "checksums.md5_hash", String.class), @TransportCompatibility(Transport.HTTP) - MEDIA_LINK("mediaLink"), + MEDIA_LINK("mediaLink", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - METADATA("metadata"), + METADATA("metadata", HashMap.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - METAGENERATION("metageneration"), + METAGENERATION("metageneration", Long.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - NAME("name"), + NAME("name", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - OWNER("owner"), + OWNER("owner", com.google.api.services.storage.model.StorageObject.Owner.class), @TransportCompatibility(Transport.HTTP) - SELF_LINK("selfLink"), + SELF_LINK("selfLink", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - SIZE("size"), + SIZE("size", java.math.BigInteger.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - STORAGE_CLASS("storageClass", "storage_class"), + STORAGE_CLASS("storageClass", "storage_class", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - TIME_DELETED("timeDeleted", "delete_time"), + TIME_DELETED("timeDeleted", "delete_time", com.google.api.client.util.DateTime.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - TIME_CREATED("timeCreated", "create_time"), + TIME_CREATED("timeCreated", "create_time", com.google.api.client.util.DateTime.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - KMS_KEY_NAME("kmsKeyName", "kms_key"), + KMS_KEY_NAME("kmsKeyName", "kms_key", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - EVENT_BASED_HOLD("eventBasedHold", "event_based_hold"), + EVENT_BASED_HOLD("eventBasedHold", "event_based_hold", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - TEMPORARY_HOLD("temporaryHold", "temporary_hold"), + TEMPORARY_HOLD("temporaryHold", "temporary_hold", String.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - RETENTION_EXPIRATION_TIME("retentionExpirationTime", "retention_expire_time"), + RETENTION_EXPIRATION_TIME( + "retentionExpirationTime", + "retention_expire_time", + com.google.api.client.util.DateTime.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - UPDATED("updated", "update_time"), + UPDATED("updated", "update_time", com.google.api.client.util.DateTime.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CUSTOM_TIME("customTime", "custom_time"), + CUSTOM_TIME("customTime", "custom_time", com.google.api.client.util.DateTime.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - TIME_STORAGE_CLASS_UPDATED("timeStorageClassUpdated", "update_storage_class_time"), + TIME_STORAGE_CLASS_UPDATED( + "timeStorageClassUpdated", + "update_storage_class_time", + com.google.api.client.util.DateTime.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CUSTOMER_ENCRYPTION("customerEncryption", "customer_encryption"), + CUSTOMER_ENCRYPTION("customerEncryption", "customer_encryption", String.class), @TransportCompatibility({Transport.HTTP}) - RETENTION("retention"), + RETENTION("retention", com.google.api.services.storage.model.StorageObject.Retention.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - SOFT_DELETE_TIME("softDeleteTime", "soft_delete_time"), + SOFT_DELETE_TIME( + "softDeleteTime", "soft_delete_time", com.google.api.client.util.DateTime.class), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - HARD_DELETE_TIME("hardDeleteTime", "hard_delete_time"); + HARD_DELETE_TIME( + "hardDeleteTime", "hard_delete_time", com.google.api.client.util.DateTime.class); static final List REQUIRED_FIELDS = ImmutableList.of(BUCKET, NAME); + private static final Map JSON_FIELD_NAME_INDEX; + + static { + ImmutableMap.Builder tmp = ImmutableMap.builder(); + for (BlobField field : values()) { + tmp.put(field.selector, field); + } + JSON_FIELD_NAME_INDEX = Utils.mapBuild(tmp); + } private final String selector; private final String grpcFieldName; + private final Class jsonClass; - BlobField(String selector) { - this(selector, selector); + BlobField(String selector, Class jsonClass) { + this(selector, selector, jsonClass); } - BlobField(String selector, String grpcFieldName) { + BlobField(String selector, String grpcFieldName, Class jsonClass) { this.selector = selector; this.grpcFieldName = grpcFieldName; + this.jsonClass = jsonClass; } @Override @@ -302,6 +367,20 @@ public String getApiaryName() { public String getGrpcName() { return grpcFieldName; } + + Class getJsonClass() { + return jsonClass; + } + + @Nullable + static BlobField lookup(NamedField nf) { + NamedField lookup = nf; + if (nf instanceof NestedNamedField) { + NestedNamedField nested = (NestedNamedField) nf; + lookup = nested.getParent(); + } + return JSON_FIELD_NAME_INDEX.get(lookup.getApiaryName()); + } } enum UriScheme { diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 5884a51e52..c01742b860 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -23,6 +23,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.Executors.callable; +import com.google.api.client.util.Data; import com.google.api.core.ApiFuture; import com.google.api.gax.paging.Page; import com.google.api.gax.retrying.ResultRetryAlgorithm; @@ -44,6 +45,8 @@ import com.google.cloud.storage.PostPolicyV4.PostConditionsV4; import com.google.cloud.storage.PostPolicyV4.PostFieldsV4; import com.google.cloud.storage.PostPolicyV4.PostPolicyV4Document; +import com.google.cloud.storage.UnifiedOpts.NamedField; +import com.google.cloud.storage.UnifiedOpts.NestedNamedField; import com.google.cloud.storage.UnifiedOpts.ObjectSourceOpt; import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt; import com.google.cloud.storage.UnifiedOpts.Opts; @@ -91,6 +94,7 @@ import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Stream; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -497,11 +501,35 @@ private static Page listBlobs( public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) { Map optionsMap = Opts.unwrap(options).resolveFrom(bucketInfo).getRpcOptions(); - if (bucketInfo.getModifiedFields().isEmpty()) { + ImmutableSet modifiedFields = bucketInfo.getModifiedFields(); + if (modifiedFields.isEmpty()) { return internalBucketGet(bucketInfo.getName(), optionsMap); } else { + com.google.api.services.storage.model.Bucket tmp = codecs.bucketInfo().encode(bucketInfo); com.google.api.services.storage.model.Bucket bucketPb = - codecs.bucketInfo().encode(bucketInfo); + new com.google.api.services.storage.model.Bucket(); + Stream.concat(modifiedFields.stream(), BucketField.REQUIRED_FIELDS.stream()) + .map( + f -> { + if (f instanceof NestedNamedField) { + return ((NestedNamedField) f).getParent(); + } else { + return f; + } + }) + .forEach( + field -> { + String jsonName = field.getApiaryName(); + if (tmp.containsKey(jsonName)) { + bucketPb.put(jsonName, tmp.get(jsonName)); + } else { + BucketField lookup = BucketField.lookup(field); + if (lookup != null) { + bucketPb.put(jsonName, Data.nullOf(lookup.getJsonClass())); + } + } + }); + ResultRetryAlgorithm algorithm = retryAlgorithmManager.getForBucketsUpdate(bucketPb, optionsMap); return run( @@ -515,7 +543,8 @@ public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) { public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { Opts opts = Opts.unwrap(options).resolveFrom(blobInfo); Map optionsMap = opts.getRpcOptions(); - boolean unmodifiedBeforeOpts = blobInfo.getModifiedFields().isEmpty(); + ImmutableSet modifiedFields = blobInfo.getModifiedFields(); + boolean unmodifiedBeforeOpts = modifiedFields.isEmpty(); BlobInfo.Builder builder = blobInfo.toBuilder(); // This is a workaround until everything is in prod for both json and grpc. @@ -523,7 +552,7 @@ public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { // request if it was modified, so that we don't send a null object in a // grpc or json request. // todo: b/308194853 - if (blobInfo.getModifiedFields().contains(BlobField.RETENTION)) { + if (modifiedFields.contains(BlobField.RETENTION)) { builder.setRetention(blobInfo.getRetention()); } BlobInfo updated = opts.blobInfoMapper().apply(builder).build(); @@ -531,7 +560,30 @@ public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { if (unmodifiedBeforeOpts && unmodifiedAfterOpts) { return internalGetBlob(blobInfo.getBlobId(), optionsMap); } else { - StorageObject pb = codecs.blobInfo().encode(updated); + StorageObject tmp = codecs.blobInfo().encode(updated); + StorageObject pb = new StorageObject(); + Stream.concat(modifiedFields.stream(), BlobField.REQUIRED_FIELDS.stream()) + .map( + f -> { + if (f instanceof NestedNamedField) { + return ((NestedNamedField) f).getParent(); + } else { + return f; + } + }) + .forEach( + field -> { + String jsonName = field.getApiaryName(); + if (tmp.containsKey(jsonName)) { + pb.put(jsonName, tmp.get(jsonName)); + } else { + BlobField lookup = BlobField.lookup(field); + if (lookup != null) { + pb.put(jsonName, Data.nullOf(lookup.getJsonClass())); + } + } + }); + ResultRetryAlgorithm algorithm = retryAlgorithmManager.getForObjectsUpdate(pb, optionsMap); return run( algorithm, diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java index 16c76ef926..15476cfcf0 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java @@ -2301,21 +2301,6 @@ public String toString() { */ @SuppressWarnings("unchecked") static final class Opts { - private static final Function, ImmutableMap> mapBuild; - - static { - Function, ImmutableMap> tmp; - // buildOrThrow was added in guava 31.0 - // if it fails, fallback to the older build() method instead. - // The behavior was the same, but the new name makes the behavior clear - try { - ImmutableMap.builder().buildOrThrow(); - tmp = ImmutableMap.Builder::buildOrThrow; - } catch (NoSuchMethodError e) { - tmp = ImmutableMap.Builder::build; - } - mapBuild = tmp; - } private final ImmutableList opts; @@ -2402,7 +2387,7 @@ Opts projectAsSource() { ImmutableMap getRpcOptions() { ImmutableMap.Builder builder = rpcOptionMapper().apply(ImmutableMap.builder()); - return (ImmutableMap) mapBuild.apply(builder); + return Utils.mapBuild(builder); } Mapper grpcMetadataMapper() { @@ -2798,7 +2783,7 @@ public String toString() { } } - private static final class NestedNamedField implements NamedField { + static final class NestedNamedField implements NamedField { private static long serialVersionUID = -7623005572810688221L; private final NamedField parent; private final NamedField child; @@ -2818,6 +2803,14 @@ public String getGrpcName() { return parent.getGrpcName() + "." + child.getGrpcName(); } + NamedField getParent() { + return parent; + } + + NamedField getChild() { + return child; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java index c24a68d4d6..6359ec5d13 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java @@ -25,6 +25,8 @@ import com.google.cloud.storage.Conversions.Codec; import com.google.cloud.storage.UnifiedOpts.NamedField; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import com.google.common.io.BaseEncoding; @@ -55,6 +57,21 @@ */ @InternalApi final class Utils { + private static final Function, ImmutableMap> mapBuild; + + static { + Function, ImmutableMap> tmp; + // buildOrThrow was added in guava 31.0 + // if it fails, fallback to the older build() method instead. + // The behavior was the same, but the new name makes the behavior clear + try { + ImmutableMap.builder().buildOrThrow(); + tmp = ImmutableMap.Builder::buildOrThrow; + } catch (NoSuchMethodError e) { + tmp = ImmutableMap.Builder::build; + } + mapBuild = tmp; + } static final DateTimeFormatter RFC_3339_DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); @@ -310,4 +327,16 @@ private static String crc32cEncode(int from) { static GrpcCallContext merge(@NonNull GrpcCallContext l, @NonNull GrpcCallContext r) { return (GrpcCallContext) l.merge(r); } + + static ImmutableList nullSafeList(@Nullable T t) { + if (t == null) { + return ImmutableList.of(); + } else { + return ImmutableList.of(t); + } + } + + static ImmutableMap mapBuild(ImmutableMap.Builder b) { + return (ImmutableMap) mapBuild.apply(b); + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/DefaultRetryHandlingBehaviorTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/DefaultRetryHandlingBehaviorTest.java index f1c37b09d4..0f90c7524f 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/DefaultRetryHandlingBehaviorTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/DefaultRetryHandlingBehaviorTest.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.net.SocketException; import java.net.SocketTimeoutException; +import java.net.UnknownHostException; import java.security.cert.CertificateException; import java.util.Arrays; import java.util.Collection; @@ -342,6 +343,7 @@ enum ThrowableCategory { IO_EXCEPTION(new IOException("no retry")), AUTH_RETRYABLE_TRUE(new RetryableException(true)), AUTH_RETRYABLE_FALSE(new RetryableException(false)), + UNKNOWN_HOST_EXCEPTION(C.UNKNOWN_HOST_EXCEPTION), ; private final Throwable throwable; @@ -415,6 +417,8 @@ private static final class C { private static final MalformedJsonException GSON_MALFORMED_EXCEPTION = new MalformedJsonException("parse-exception"); private static final IOException IO_PREMATURE_EOF = new IOException("Premature EOF"); + private static final UnknownHostException UNKNOWN_HOST_EXCEPTION = + new UnknownHostException("fake.fake"); private static HttpResponseException newHttpResponseException( int httpStatusCode, String name) { @@ -1065,6 +1069,16 @@ private static ImmutableList getAllCases() { ThrowableCategory.AUTH_RETRYABLE_FALSE, HandlerCategory.NONIDEMPOTENT, ExpectRetry.NO, + Behavior.SAME), + new Case( + ThrowableCategory.UNKNOWN_HOST_EXCEPTION, + HandlerCategory.IDEMPOTENT, + ExpectRetry.YES, + Behavior.DEFAULT_MORE_PERMISSIBLE), + new Case( + ThrowableCategory.UNKNOWN_HOST_EXCEPTION, + HandlerCategory.NONIDEMPOTENT, + ExpectRetry.NO, Behavior.SAME)) .build(); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcMetricsTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcMetricsTest.java new file mode 100644 index 0000000000..f1a406ca6c --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcMetricsTest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.SingleBackend; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@SingleBackend(Backend.PROD) +public class ITGrpcMetricsTest { + @Test + public void testGrpcMetrics() { + GrpcStorageOptions grpcStorageOptions = StorageOptions.grpc().build(); + assertThat( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "storage.googleapis.com:443", "storage.googleapis.com")) + .isEqualTo("monitoring.googleapis.com:443"); + + SdkMeterProvider provider = + OpenTelemetryBootstrappingUtils.createMeterProvider( + "monitoring.googleapis.com:443", grpcStorageOptions.getProjectId(), false); + + /* + * SDKMeterProvider doesn't expose the relevant fields we want to test, but they are present in + * the String representation, so we'll check that instead. Most of the resources are auto-set, + * and will depend on environment, which could cause flakes to check. We're only responsible for + * setting the project ID, endpoint, and Histogram boundaries, so we'll just check those + */ + String result = provider.toString(); + + // What the project ID will be will depend on the environment, so we just make sure it's present + // and not null/empty + assertThat(result.contains("project_id")); + assertThat(result).doesNotContain("project_id=\"\""); + assertThat(result).doesNotContain("project_id=null"); + + // This is the check for the Seconds histogram boundary. We can't practically check for every + // boundary, + // but if *any* are present, that means they're different from the results and we successfully + // set them + assertThat(result).contains("1.2"); + + // This is the check for the Size boundary + assertThat(result).contains("131072"); + } + + @Test + public void testGrpcMetrics_universeDomain() { + assertThat("monitoring.my-universe-domain.com:443") + .isEqualTo( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "storage.my-universe-domain.com:443", "my-universe-domain.com")); + } + + @Test + public void testGrpcMetrics_private() { + assertThat("private.googleapis.com:443") + .isEqualTo( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "private.googleapis.com:443", null)); + } + + @Test + public void testGrpcMetrics_restricted() { + assertThat("restricted.googleapis.com:443") + .isEqualTo( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "restricted.googleapis.com:443", null)); + } +} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcTest.java index 4d6f0aa08e..3ac87cad0f 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcTest.java @@ -251,12 +251,11 @@ public void lockBucketRetentionPolicy() { @Test public void testGrpcUniverseDomainMatchesHost() throws Exception { - Storage storage = - StorageOptions.grpc().setUniverseDomain("my-universe-domain.com").build().getService(); + StorageOptions storageOptions = + StorageOptions.grpc().setUniverseDomain("my-universe-domain.com").build(); assertAll( - () -> assertThat(storage.getOptions().getUniverseDomain().equals("my-universe-domain.com")), + () -> assertThat(storageOptions.getUniverseDomain().equals("my-universe-domain.com")), () -> - assertThat( - storage.getOptions().getHost().equals("https://storage.my-universe-domain.com"))); + assertThat(storageOptions.getHost().equals("https://storage.my-universe-domain.com"))); } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITJsonPatchTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITJsonPatchTest.java new file mode 100644 index 0000000000..bb2bdfe691 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITJsonPatchTest.java @@ -0,0 +1,147 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.cloud.storage.TestUtils.assertAll; +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.BucketInfo.LifecycleRule; +import com.google.cloud.storage.BucketInfo.LifecycleRule.LifecycleAction; +import com.google.cloud.storage.BucketInfo.LifecycleRule.LifecycleCondition; +import com.google.cloud.storage.Cors; +import com.google.cloud.storage.Cors.Origin; +import com.google.cloud.storage.HttpMethod; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobTargetOption; +import com.google.cloud.storage.Storage.BucketTargetOption; +import com.google.cloud.storage.TransportCompatibility.Transport; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.CrossRun; +import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.registry.Generator; +import com.google.common.collect.ImmutableList; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@CrossRun( + backends = {Backend.PROD}, + transports = {Transport.HTTP, Transport.GRPC}) +public final class ITJsonPatchTest { + + @Inject public Storage storage; + @Inject public BucketInfo bucket; + @Inject public Generator generator; + + @Test + public void object() throws Exception { + String cacheControl = "max-age=60"; + String contentDisposition = "attachment"; + String contentEncoding = "identity"; + String contentLanguage = "en-US"; + String contentType = "text/plain"; + BlobInfo info = + BlobInfo.newBuilder(bucket, generator.randomObjectName()) + .setCacheControl(cacheControl) + .setContentDisposition(contentDisposition) + .setContentEncoding(contentEncoding) + .setContentLanguage(contentLanguage) + .setContentType(contentType) + .build(); + + Blob gen1 = storage.create(info); + assertAll( + () -> assertThat(gen1.getCacheControl()).isEqualTo(cacheControl), + () -> assertThat(gen1.getContentDisposition()).isEqualTo(contentDisposition), + () -> assertThat(gen1.getContentEncoding()).isEqualTo(contentEncoding), + () -> assertThat(gen1.getContentLanguage()).isEqualTo(contentLanguage), + () -> assertThat(gen1.getContentType()).isEqualTo(contentType)); + BlobInfo update = + gen1.toBuilder() + .setCacheControl(null) + .setContentDisposition(null) + .setContentEncoding(null) + .setContentLanguage(null) + .setContentType(null) + .build(); + Blob gen2 = + storage.update(update, BlobTargetOption.metagenerationMatch(gen1.getMetageneration())); + assertAll( + () -> assertThat(gen2.getCacheControl()).isAnyOf("", null), + () -> assertThat(gen2.getContentDisposition()).isAnyOf("", null), + () -> assertThat(gen2.getContentEncoding()).isAnyOf("", null), + () -> assertThat(gen2.getContentLanguage()).isAnyOf("", null), + () -> assertThat(gen2.getContentType()).isAnyOf("", null)); + } + + @Test + public void bucket() throws Exception { + ImmutableList lifecycleRules = + ImmutableList.of( + new LifecycleRule( + LifecycleAction.newDeleteAction(), + LifecycleCondition.newBuilder() + .setMatchesPrefix(ImmutableList.of("blahblahblah")) + .build())); + ImmutableList cors = + ImmutableList.of( + Cors.newBuilder() + .setMaxAgeSeconds(300) + .setMethods(ImmutableList.of(HttpMethod.GET)) + .setOrigins(ImmutableList.of(Origin.any())) + .setResponseHeaders(ImmutableList.of("blah2blah")) + .build()); + String indexPage = "index.html"; + String notFoundPage = "404.html"; + BucketInfo info = + BucketInfo.newBuilder(generator.randomBucketName()) + .setLifecycleRules(lifecycleRules) + .setCors(cors) + .setIndexPage(indexPage) + .setNotFoundPage(notFoundPage) + .build(); + + try (TemporaryBucket tmpBucket = + TemporaryBucket.newBuilder().setBucketInfo(info).setStorage(storage).build()) { + BucketInfo gen1 = tmpBucket.getBucket(); + + assertAll( + () -> assertThat(gen1.getLifecycleRules()).isEqualTo(lifecycleRules), + () -> assertThat(gen1.getCors()).isEqualTo(cors), + () -> assertThat(gen1.getIndexPage()).isEqualTo(indexPage), + () -> assertThat(gen1.getNotFoundPage()).isEqualTo(notFoundPage)); + BucketInfo update = + gen1.toBuilder() + .setLifecycleRules(ImmutableList.of()) + .setCors(ImmutableList.of()) + .setIndexPage(null) + .setNotFoundPage(null) + .build(); + Bucket gen2 = storage.update(update, BucketTargetOption.metagenerationMatch()); + assertAll( + () -> assertThat(gen2.getLifecycleRules()).isAnyOf(ImmutableList.of(), null), + () -> assertThat(gen2.getCors()).isAnyOf(ImmutableList.of(), null), + () -> assertThat(gen2.getIndexPage()).isAnyOf("", null), + () -> assertThat(gen2.getNotFoundPage()).isAnyOf("", null)); + } + } +} diff --git a/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile b/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile index a18e9da047..3bc94b3144 100644 --- a/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile +++ b/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile @@ -1 +1 @@ -FROM gcr.io/cloud-devrel-public-resources/storage-testbench:v0.44.0 +FROM gcr.io/cloud-devrel-public-resources/storage-testbench:v0.45.0 diff --git a/grpc-google-cloud-storage-control-v2/pom.xml b/grpc-google-cloud-storage-control-v2/pom.xml index b102687fca..81f9a2710a 100644 --- a/grpc-google-cloud-storage-control-v2/pom.xml +++ b/grpc-google-cloud-storage-control-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-storage-control-v2 - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT grpc-google-cloud-storage-control-v2 GRPC library for google-cloud-storage com.google.cloud google-cloud-storage-parent - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT diff --git a/grpc-google-cloud-storage-v2/pom.xml b/grpc-google-cloud-storage-v2/pom.xml index 5256d6ce86..5301429f01 100644 --- a/grpc-google-cloud-storage-v2/pom.xml +++ b/grpc-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-storage-v2 - 2.40.2-alpha-SNAPSHOT + 2.41.1-alpha-SNAPSHOT grpc-google-cloud-storage-v2 GRPC library for grpc-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT diff --git a/pom.xml b/pom.xml index 6481121d96..17372b4270 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT Storage Parent https://github.com/googleapis/java-storage @@ -14,7 +14,7 @@ com.google.cloud sdk-platform-java-config - 3.32.0 + 3.33.0 @@ -54,7 +54,7 @@ UTF-8 github google-cloud-storage-parent - 3.25.0 + 3.31.0 @@ -65,10 +65,61 @@ ${google-cloud-shared-dependencies.version} pom import + + + + io.opentelemetry.semconv + opentelemetry-semconv + 1.25.0-alpha + + + com.google.cloud.opentelemetry + exporter-metrics + 0.31.0 + + + io.opentelemetry.semconv + opentelemetry-semconv + + + + + com.google.cloud.opentelemetry + detector-resources + 0.27.0-alpha + + + io.opentelemetry.semconv + opentelemetry-semconv + + + + + io.opentelemetry.instrumentation + opentelemetry-resources + 2.6.0-alpha + + + io.opentelemetry.semconv + opentelemetry-semconv + + + + + io.opentelemetry.contrib + opentelemetry-gcp-resources + 1.37.0-alpha - io.grpc - * + io.opentelemetry.semconv + opentelemetry-semconv @@ -76,7 +127,7 @@ com.google.cloud google-cloud-storage - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT com.google.apis @@ -86,7 +137,7 @@ com.google.cloud google-cloud-pubsub - 1.131.0 + 1.132.0 test @@ -98,32 +149,32 @@ com.google.api.grpc proto-google-cloud-storage-v2 - 2.40.2-alpha-SNAPSHOT + 2.41.1-alpha-SNAPSHOT com.google.api.grpc grpc-google-cloud-storage-v2 - 2.40.2-alpha-SNAPSHOT + 2.41.1-alpha-SNAPSHOT com.google.api.grpc gapic-google-cloud-storage-v2 - 2.40.2-alpha-SNAPSHOT + 2.41.1-alpha-SNAPSHOT com.google.api.grpc grpc-google-cloud-storage-control-v2 - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT com.google.api.grpc proto-google-cloud-storage-control-v2 - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT com.google.cloud google-cloud-storage-control - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT com.google.cloud diff --git a/proto-google-cloud-storage-control-v2/pom.xml b/proto-google-cloud-storage-control-v2/pom.xml index ff2abbc939..348a461017 100644 --- a/proto-google-cloud-storage-control-v2/pom.xml +++ b/proto-google-cloud-storage-control-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-storage-control-v2 - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT proto-google-cloud-storage-control-v2 Proto library for proto-google-cloud-storage-control-v2 com.google.cloud google-cloud-storage-parent - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT diff --git a/proto-google-cloud-storage-v2/pom.xml b/proto-google-cloud-storage-v2/pom.xml index 193645a5d8..6179707e9f 100644 --- a/proto-google-cloud-storage-v2/pom.xml +++ b/proto-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-storage-v2 - 2.40.2-alpha-SNAPSHOT + 2.41.1-alpha-SNAPSHOT proto-google-cloud-storage-v2 PROTO library for proto-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT diff --git a/renovate.json b/renovate.json index c7467c6b72..42380a2b69 100644 --- a/renovate.json +++ b/renovate.json @@ -111,6 +111,16 @@ ], "semanticCommitType": "test", "semanticCommitScope": "deps" + }, + { + "groupName": "OpenTelemetry extended dependencies", + "packagePatterns": [ + "^io.opentelemetry.semconv:", + "^io.opentelemetry.instrumentation:", + "^io.opentelemetry.contrib:", + "^com.google.cloud.opentelemetry:", + "$com.google.cloud.opentelemetry:shared-resourcemapping" + ] } ], "semanticCommits": true, diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index d78aef5226..8c13882628 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -30,12 +30,12 @@ com.google.cloud google-cloud-storage - 2.40.1 + 2.41.0 com.google.cloud google-cloud-storage-control - 2.40.2-SNAPSHOT + 2.41.0 @@ -66,15 +66,9 @@ com.google.cloud google-cloud-pubsub - 1.131.0 + 1.132.0 test - - com.google.cloud - google-cloud-storage-control - 2.40.1 - compile - diff --git a/samples/native-image-sample/README.md b/samples/native-image-sample/README.md deleted file mode 100644 index 092cdc0591..0000000000 --- a/samples/native-image-sample/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# Storage Sample Application with Native Image - -The Storage sample application demonstrates some common operations with Google Cloud Storage and is compatible with Native Image compilation. - -## Setup Instructions - -You will need to follow these prerequisite steps in order to run these samples: - -1. If you have not already, [create a Google Cloud Platform Project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project). - -2. Install the [Google Cloud SDK](https://cloud.google.com/sdk/) which will allow you to run the sample with your project's credentials. - - Once installed, log in with Application Default Credentials using the following command: - - ``` - gcloud auth application-default login - ``` - - **Note:** Authenticating with Application Default Credentials is convenient to use during development, but we recommend [alternate methods of authentication](https://cloud.google.com/docs/authentication/production) during production use. - -3. Install the GraalVM compiler. - - You can follow the [official installation instructions](https://www.graalvm.org/docs/getting-started/#install-graalvm) from the GraalVM website. - After following the instructions, ensure that you install the Native Image extension installed by running: - - ``` - gu install native-image - ``` - - Once you finish following the instructions, verify that the default version of Java is set to the GraalVM version by running `java -version` in a terminal. - - You will see something similar to the below output: - - ``` - $ java -version - - openjdk version "11.0.7" 2020-04-14 - OpenJDK Runtime Environment GraalVM CE 20.1.0 (build 11.0.7+10-jvmci-20.1-b02) - OpenJDK 64-Bit Server VM GraalVM CE 20.1.0 (build 11.0.7+10-jvmci-20.1-b02, mixed mode, sharing) - ``` - -4. [Enable the Cloud Storage APIs](https://console.cloud.google.com/apis/api/storage.googleapis.com). - -### Run with Native Image Compilation - -Navigate to this directory in a new terminal. - -1. Compile the application using the Native Image Compiler. This step may take a few minutes. - - ``` - mvn package -P native -DskipTests - ``` - -2. Run the application: - - ``` - ./target/native-image-sample - ``` - -3. The application will run through basic Cloud Storage operations of creating, reading, and deleting Cloud Storage resources. - - You can manually manage Cloud Storage resources through [Google Cloud Console](https://console.cloud.google.com/storage) to verify that the resources are cleaned up. - - ``` - Creating bucket nativeimage-sample-bucket-7221f161-688c-4a7a-9120-8900d20f0802 - Write file to bucket. - Reading the file that was written... - Successfully wrote to file: Hello World! - Cleaning up resources... - Deleted file nativeimage-sample-file-5d927aaf-cb03-41de-8383-696733893db5 - Deleted bucket nativeimage-sample-bucket-7221f161-688c-4a7a-9120-8900d20f0802 - ``` - -## Sample Integration test with Native Image Support - -In order to run the sample integration test as a native image, call the following command: - - ``` - mvn test -Pnative - ``` \ No newline at end of file diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml deleted file mode 100644 index e25c7e101e..0000000000 --- a/samples/native-image-sample/pom.xml +++ /dev/null @@ -1,174 +0,0 @@ - - - 4.0.0 - com.google.cloud - native-image-sample - Native Image Sample - https://github.com/googleapis/java-storage - - - - com.google.cloud.samples - shared-configuration - 1.2.0 - - - - - - 1.8 - 1.8 - UTF-8 - - - - - - com.google.cloud - libraries-bom - 26.43.0 - pom - import - - - - - - - com.google.cloud - google-cloud-core - - - com.google.cloud - google-cloud-storage - - - - junit - junit - 4.13.2 - test - - - com.google.truth - truth - 1.4.4 - test - - - com.google.cloud - google-cloud-pubsub - 1.131.0 - test - - - com.google.cloud - google-cloud-storage-control - 2.40.2-SNAPSHOT - compile - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.4.2 - - - - true - dependency-jars/ - com.example.storage.NativeImageStorageSample - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.7.1 - - - copy-dependencies - package - - copy-dependencies - - - - ${project.build.directory}/dependency-jars/ - - - - - - - - - - - - native - - - - org.junit.vintage - junit-vintage-engine - 5.10.3 - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.3.1 - - - **/*IT - - - - - org.graalvm.buildtools - native-maven-plugin - 0.10.2 - true - - com.example.storage.NativeImageStorageSample - - --no-fallback - --no-server - - - - - build-native - - build - test - - package - - - test-native - - test - - test - - - - - - - - diff --git a/samples/native-image-sample/src/main/java/com/example/storage/NativeImageStorageSample.java b/samples/native-image-sample/src/main/java/com/example/storage/NativeImageStorageSample.java deleted file mode 100644 index ce4cf6b5ef..0000000000 --- a/samples/native-image-sample/src/main/java/com/example/storage/NativeImageStorageSample.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.storage; - -import com.google.cloud.BatchResult.Callback; -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.BlobId; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.BucketInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageBatch; -import com.google.cloud.storage.StorageException; -import com.google.cloud.storage.StorageOptions; -import java.nio.charset.StandardCharsets; -import java.util.UUID; - -/** Sample Storage application compiled with Native Image. */ -public class NativeImageStorageSample { - - static String BUCKET_NAME = "nativeimage-sample-bucket-" + UUID.randomUUID(); - static String FILENAME = "nativeimage-sample-file-" + UUID.randomUUID(); - - /** Runs the storage sample application. */ - public static void main(String[] args) { - - Storage storageClient = StorageOptions.getDefaultInstance().getService(); - - try { - createBucket(storageClient, BUCKET_NAME); - createFile(storageClient, BUCKET_NAME, FILENAME); - runBatchOperations(storageClient, BUCKET_NAME, FILENAME); - } finally { - System.out.println("Deleting resources."); - storageClient.delete(BUCKET_NAME, FILENAME); - storageClient.delete(BUCKET_NAME); - } - } - - private static void runBatchOperations( - Storage storageClient, String bucketName, String fileName) { - BlobId blobId = BlobId.of(bucketName, fileName); - - StorageBatch batch = storageClient.batch(); - batch - .update(BlobInfo.newBuilder(blobId).build()) - .notify( - new Callback() { - @Override - public void success(Blob blob) { - System.out.println("Batch update succeeded on " + fileName); - } - - @Override - public void error(StorageException e) { - System.out.println("Batch update failed with cause: " + e); - } - }); - - batch.submit(); - } - - private static void createBucket(Storage storageClient, String bucketName) { - BucketInfo bucketInfo = BucketInfo.newBuilder(bucketName).setLocation("us-east1").build(); - storageClient.create(bucketInfo); - System.out.println("Created bucket " + bucketName); - } - - private static void createFile(Storage storageClient, String bucketName, String fileName) { - BlobInfo blobInfo = - BlobInfo.newBuilder(bucketName, fileName).setContentType("text/plain").build(); - storageClient.create(blobInfo, "Hello World!".getBytes(StandardCharsets.UTF_8)); - System.out.println("Created file " + blobInfo.getName()); - - Blob blob = storageClient.get(bucketName, fileName); - String content = new String(blob.getContent(), StandardCharsets.UTF_8); - System.out.println("Successfully wrote to file: " + content); - } -} diff --git a/samples/native-image-sample/src/test/java/com/example/storage/NativeImageStorageSampleIT.java b/samples/native-image-sample/src/test/java/com/example/storage/NativeImageStorageSampleIT.java deleted file mode 100644 index 5098402647..0000000000 --- a/samples/native-image-sample/src/test/java/com/example/storage/NativeImageStorageSampleIT.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.storage; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.cloud.testing.junit4.StdOutCaptureRule; -import org.junit.Rule; -import org.junit.Test; - -public class NativeImageStorageSampleIT { - - @Rule public StdOutCaptureRule stdOut = new StdOutCaptureRule(); - - @Test - public void createAndReadStorageResources() { - NativeImageStorageSample.main(new String[] {}); - assertThat(stdOut.getCapturedOutputAsUtf8String()) - .contains("Created bucket " + NativeImageStorageSample.BUCKET_NAME); - assertThat(stdOut.getCapturedOutputAsUtf8String()) - .contains("Created file " + NativeImageStorageSample.FILENAME); - assertThat(stdOut.getCapturedOutputAsUtf8String()) - .contains("Successfully wrote to file: Hello World!"); - } -} diff --git a/samples/pom.xml b/samples/pom.xml index 7c25246774..cb1009a912 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -32,7 +32,6 @@ install-without-bom snapshot snippets - native-image-sample diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 848c09dfcc..f38a946fbe 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,12 +28,12 @@ com.google.cloud google-cloud-storage - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT com.google.cloud google-cloud-storage-control - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT compile @@ -58,7 +58,7 @@ com.google.cloud google-cloud-pubsub - 1.131.0 + 1.132.0 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index de1be98b7c..125ffd9775 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -31,7 +31,7 @@ com.google.cloud libraries-bom - 26.43.0 + 26.44.0 pom import @@ -76,8 +76,10 @@ com.google.cloud google-cloud-pubsub - 1.131.0 + 1.132.0 test + + diff --git a/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java b/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java index da9fd12b1b..60d918e842 100644 --- a/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java +++ b/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java @@ -70,7 +70,6 @@ public void testQuickstartGrpc() throws Exception { assertThat(got).contains(String.format("Bucket %s created.", bucketName)); } - @Ignore("https://github.com/googleapis/java-storage/issues/2612") @Test public void testQuickstartGrpcDp() throws Exception { QuickstartGrpcDpSample.main(bucketName); diff --git a/storage-shared-benchmarking/pom.xml b/storage-shared-benchmarking/pom.xml index d51510c251..d01efdcfb6 100644 --- a/storage-shared-benchmarking/pom.xml +++ b/storage-shared-benchmarking/pom.xml @@ -10,7 +10,7 @@ com.google.cloud google-cloud-storage-parent - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT @@ -31,7 +31,7 @@ com.google.cloud google-cloud-storage - 2.40.2-SNAPSHOT + 2.41.1-SNAPSHOT tests diff --git a/versions.txt b/versions.txt index f69020a214..3e008150c9 100644 --- a/versions.txt +++ b/versions.txt @@ -1,10 +1,10 @@ # Format: # module:released-version:current-version -google-cloud-storage:2.40.1:2.40.2-SNAPSHOT -gapic-google-cloud-storage-v2:2.40.1-alpha:2.40.2-alpha-SNAPSHOT -grpc-google-cloud-storage-v2:2.40.1-alpha:2.40.2-alpha-SNAPSHOT -proto-google-cloud-storage-v2:2.40.1-alpha:2.40.2-alpha-SNAPSHOT -google-cloud-storage-control:2.40.1:2.40.2-SNAPSHOT -proto-google-cloud-storage-control-v2:2.40.1:2.40.2-SNAPSHOT -grpc-google-cloud-storage-control-v2:2.40.1:2.40.2-SNAPSHOT +google-cloud-storage:2.41.0:2.41.1-SNAPSHOT +gapic-google-cloud-storage-v2:2.41.0-alpha:2.41.1-alpha-SNAPSHOT +grpc-google-cloud-storage-v2:2.41.0-alpha:2.41.1-alpha-SNAPSHOT +proto-google-cloud-storage-v2:2.41.0-alpha:2.41.1-alpha-SNAPSHOT +google-cloud-storage-control:2.41.0:2.41.1-SNAPSHOT +proto-google-cloud-storage-control-v2:2.41.0:2.41.1-SNAPSHOT +grpc-google-cloud-storage-control-v2:2.41.0:2.41.1-SNAPSHOT