Upgrade GCS SDK to 2.13.1 (#92327) (#92527)

Upgrade to the latest version of the SDK that doesn't have known CVEs
and builds w/o complaining. Since 2.2.0 the automatic retry behaviour
has changed and the old behaviour can still be used as
LegacyStorageRetryStrategy. The default retry strategy would cause some
test failures, and therefore we'd need to explicitly set a retry
strategy.

Relates #92474

docs/changelog/92327.yaml

@@ -0,0 +1,5 @@
+pr: 92327
+summary: Upgrade GCS SDK to 2.13.1
+area: Snapshot/Restore
+type: upgrade
+issues: []

gradle/verification-metadata.xml

@@ -454,11 +454,26 @@
             <sha256 value="197b23f2d22324a923e5d643d1c71c51442ae7b9b166fe6ecd5ad9b6916689b5" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.api" name="api-common" version="2.3.1">
+         <artifact name="api-common-2.3.1.jar">
+            <sha256 value="9a8c0f3a638eac791496709b7770c39bda9e56de50728f122237c813bb21f1ee" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.api" name="gax" version="2.0.0">
          <artifact name="gax-2.0.0.jar">
             <sha256 value="37974e31bf9ef5119b0411f83aaa209d4edb4fb10646eb6eaff68e7771a64e50" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.api" name="gax" version="2.20.1">
+         <artifact name="gax-2.20.1.jar">
+            <sha256 value="e6588fc1786557fda95024a040b32106d58ffe72fab0933200c4d3c6197a4fe1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.google.api" name="gax-httpjson" version="0.105.1">
+         <artifact name="gax-httpjson-0.105.1.jar">
+            <sha256 value="4b7e1135eb4a97bce9d9d8c56128c5c30594dc2bebf26c9851ac582d2b43b2db" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.api" name="gax-httpjson" version="0.85.0">
          <artifact name="gax-httpjson-0.85.0.jar">
             <sha256 value="e0b0411e42c859e0fab4b092a3429aea2a71b2c2ebbbc24d719b9e762be97c35" origin="Generated by Gradle"/>
@@ -474,16 +489,31 @@
             <sha256 value="24714add0c0248730a0e705448bbc91a5c68dbc0dc9f7fa0c97366bc7b275e23" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.api-client" name="google-api-client" version="2.1.1">
+         <artifact name="google-api-client-2.1.1.jar">
+            <sha256 value="05d846e83a2924c2a6facbd72817228fee68bb522b818b9804b26a712f678bf3" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.api.grpc" name="proto-google-common-protos" version="2.3.2">
          <artifact name="proto-google-common-protos-2.3.2.jar">
             <sha256 value="99aa86a5e52dff58be410b5c30905b2ea9a2c995a87e237f160590953c661b6d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.api.grpc" name="proto-google-common-protos" version="2.9.6">
+         <artifact name="proto-google-common-protos-2.9.6.jar">
+            <sha256 value="84e715d92500fd0d44408f58713d5d70ae857d022de3d66d500fbf27fae88103" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.api.grpc" name="proto-google-iam-v1" version="1.0.14">
          <artifact name="proto-google-iam-v1-1.0.14.jar">
             <sha256 value="e722e8d67172a87ad1b5bd3ba02c678484958190c49c059494453b07e7ea5dca" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.api.grpc" name="proto-google-iam-v1" version="1.6.2">
+         <artifact name="proto-google-iam-v1-1.6.2.jar">
+            <sha256 value="4fce59633b8a9871614231d213992e60e9b50e12f822c33d7e82a79c90addf25" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.apis" name="google-api-services-compute" version="v1-rev160-1.23.0">
          <artifact name="google-api-services-compute-v1-rev160-1.23.0.jar">
             <sha256 value="135f7053fc6eae0fadf587a110254d4f325dd46f3089d120492b599900ab857f" origin="Generated by Gradle"/>
@@ -494,31 +524,61 @@
             <sha256 value="c23beb05bb842abed14c8fd75a3e6b7a2f51b0d3ece1903a9c5fd3ab770ad93e" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.apis" name="google-api-services-storage" version="v1-rev20220705-2.0.0">
+         <artifact name="google-api-services-storage-v1-rev20220705-2.0.0.jar">
+            <sha256 value="c6027d6c58089402620ce338351ca2efd414f813007844bc252ceb5f47789f4f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.auth" name="google-auth-library-credentials" version="1.0.0">
          <artifact name="google-auth-library-credentials-1.0.0.jar">
             <sha256 value="8b4e4bf2c89f72e9fdcbbd442c931e17cba01e84f058fd61e470bd19f8a92cc1" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.auth" name="google-auth-library-credentials" version="1.11.0">
+         <artifact name="google-auth-library-credentials-1.11.0.jar">
+            <sha256 value="2f47d7d2126576f13c22b7442e97593e9e6285adbd6d4c8dd779d87a8cbb8270" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.auth" name="google-auth-library-oauth2-http" version="1.0.0">
          <artifact name="google-auth-library-oauth2-http-1.0.0.jar">
             <sha256 value="7cae6c73368659ac7a6b46018070f3d7980acd5bfaaa91b840f2e869cef1f07e" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.auth" name="google-auth-library-oauth2-http" version="1.11.0">
+         <artifact name="google-auth-library-oauth2-http-1.11.0.jar">
+            <sha256 value="9fc1548aacc781367b5ca8da8e053018929d71af7303f6752952ddcb6f9b50c4" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.cloud" name="google-cloud-core" version="2.0.2">
          <artifact name="google-cloud-core-2.0.2.jar">
             <sha256 value="364be3e5617c973a4fa8ebe2a56aa8eaec9277f5dba7d033cc37b71b00025e45" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.cloud" name="google-cloud-core" version="2.8.28">
+         <artifact name="google-cloud-core-2.8.28.jar">
+            <sha256 value="f775b5dc6b6478d07be13c64b3aa8ee0eded5e2571ff737e86ed05958cce4e92" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.cloud" name="google-cloud-core-http" version="2.0.2">
          <artifact name="google-cloud-core-http-2.0.2.jar">
             <sha256 value="087f0bc4d8de1daed73acb631b68c254dcf7993379110c6edf7bc7438d0e52ca" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.cloud" name="google-cloud-core-http" version="2.8.28">
+         <artifact name="google-cloud-core-http-2.8.28.jar">
+            <sha256 value="96cd89a23a40b64baa28cba92676fba67d16cfbb1a7085827a1852a71c2dce46" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.cloud" name="google-cloud-storage" version="1.118.1">
          <artifact name="google-cloud-storage-1.118.1.jar">
             <sha256 value="21d675edc7caa02426290ed631b453004e7ece91bc811209855491263ac35f69" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.cloud" name="google-cloud-storage" version="2.13.1">
+         <artifact name="google-cloud-storage-2.13.1.jar">
+            <sha256 value="5fee796ca3c328aabe1b96adfbb264714e82b3ceee2971a83732c8cada12a9b6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.code.findbugs" name="jsr305" version="1.3.9">
          <artifact name="jsr305-1.3.9.jar">
             <sha256 value="905721a0eea90a81534abb7ee6ef4ea2e5e645fa1def0a5cd88402df1b46c9ed" origin="Generated by Gradle"/>
@@ -534,6 +594,11 @@
             <sha256 value="766ad2a0783f2687962c8ad74ceecc38a28b9f72a2d085ee438b7813e928d0c7" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.code.gson" name="gson" version="2.10">
+         <artifact name="gson-2.10.jar">
+            <sha256 value="0cdd163ce3598a20fc04eee71b140b24f6f2a3b35f0a499dbbdd9852e83fbfaf" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.code.gson" name="gson" version="2.2.4">
          <artifact name="gson-2.2.4.jar">
             <sha256 value="c0328cd07ca9e363a5acd00c1cf4afe8cf554bd6d373834981ba05cebec687fb" origin="Generated by Gradle"/>
@@ -644,16 +709,31 @@
             <sha256 value="6ab4967860927f051dd7f2b575e4b7a8ef35f367dc2e3408073aa4dee328e227" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.http-client" name="google-http-client" version="1.42.3">
+         <artifact name="google-http-client-1.42.3.jar">
+            <sha256 value="e395dd1765e3e6bceb0c610706bcf4128de84bd6e65cf1d4adbf998b4114161c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.http-client" name="google-http-client-appengine" version="1.39.2">
          <artifact name="google-http-client-appengine-1.39.2.jar">
             <sha256 value="f6320dc0db035aff6eab2e28748560da718ab465fbc3481b653ea3f27aa5cac9" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.http-client" name="google-http-client-appengine" version="1.42.3">
+         <artifact name="google-http-client-appengine-1.42.3.jar">
+            <sha256 value="3a876b1f704ac6d8500b8430b288dbce78c52dc18c43c008a587a7377bb81e2e" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.http-client" name="google-http-client-gson" version="1.39.2">
          <artifact name="google-http-client-gson-1.39.2.jar">
             <sha256 value="cf81944fd41fe0daa869b913bd0aaaef0e4d8fe6774a195540f454ecd3cf50b6" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.http-client" name="google-http-client-gson" version="1.42.3">
+         <artifact name="google-http-client-gson-1.42.3.jar">
+            <sha256 value="8196efaa89c5f73b00b2b48edad02fcd78524259407c37ab1860737988545cee" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.http-client" name="google-http-client-jackson2" version="1.23.0">
          <artifact name="google-http-client-jackson2-1.23.0.jar">
             <sha256 value="2527e9d1eac4822ba94f6fffe4f0b6d6e757e4a33ece5406674f60b7f6494b06" origin="Generated by Gradle"/>
@@ -664,6 +744,11 @@
             <sha256 value="74a493a7ba25e1dfb3d3a4ddf713306d5a3786db00a3489d59061dc8cff321b1" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.http-client" name="google-http-client-jackson2" version="1.42.3">
+         <artifact name="google-http-client-jackson2-1.42.3.jar">
+            <sha256 value="b03a79f38e924e2eeaee673acf314626825284cb39948d06c2948ad083b4d089" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.inject" name="guice" version="3.0">
          <artifact name="guice-3.0.jar">
             <sha256 value="1a59d0421ffd355cc0b70b42df1c2e9af744c8a2d0c92da379f5fca2f07f1d22" origin="Generated by Gradle"/>
@@ -724,6 +809,11 @@
             <sha256 value="ad275e75ee79e6c6388198dcb9acf0db2edee64782e11b508f379c3a2a17d168" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.protobuf" name="protobuf-java" version="3.21.7">
+         <artifact name="protobuf-java-3.21.7.jar">
+            <sha256 value="a204ec68748a7b26351ae37a311e8de468f248d1916d5f8dbe812c1289d0a0ff" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.protobuf" name="protobuf-java" version="3.4.0">
          <artifact name="protobuf-java-3.4.0.jar">
             <sha256 value="dce7e66b32456a1b1198da0caff3a8acb71548658391e798c79369241e6490a4" origin="Generated by Gradle"/>
@@ -1254,6 +1344,11 @@
             <sha256 value="97fe70b4e7da86ed4fe3deb16d83fc89c7e5b764ebac8c9dd5d51dd4b5aa920c" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.grpc" name="grpc-context" version="1.49.2">
+         <artifact name="grpc-context-1.49.2.jar">
+            <sha256 value="7aaf3ad732a281be51fe0895acd4538b96e1d7e3b527bce54b651770b8ded174" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.micrometer" name="micrometer-core" version="1.7.2">
          <artifact name="micrometer-core-1.7.2.jar">
             <sha256 value="8c63cab4c5f8f7df25af6e4c15fcab151df2fb1188b37197923acbb9bfb409da" origin="Generated by Gradle"/>
@@ -1474,11 +1569,21 @@
             <sha256 value="0c1723f3f6d3061323845ce8b88b35fdda500812e0a75b8eb5fcc4ad8c871a95" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.opencensus" name="opencensus-api" version="0.31.1">
+         <artifact name="opencensus-api-0.31.1.jar">
+            <sha256 value="f1474d47f4b6b001558ad27b952e35eda5cc7146788877fc52938c6eba24b382" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.opencensus" name="opencensus-contrib-http-util" version="0.28.0">
          <artifact name="opencensus-contrib-http-util-0.28.0.jar">
             <sha256 value="49c3db2a29f1fdb2f73928cbea969bd1d40fab7cc5bb6273022babd96f7a789b" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.opencensus" name="opencensus-contrib-http-util" version="0.31.1">
+         <artifact name="opencensus-contrib-http-util-0.31.1.jar">
+            <sha256 value="3ea995b55a4068be22989b70cc29a4d788c2d328d1d50613a7a9afd13fdd2d0a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.ous" name="jtoml" version="2.0.0">
          <artifact name="jtoml-2.0.0.jar">
             <sha256 value="3cabdae2244c999addebb8c31ae452fbdc874b4f26a163539954b8eeb5d6acc6" origin="Generated by Gradle"/>
@@ -3869,6 +3974,11 @@
             <sha256 value="4342ee04d87040f71b0aa9188ee960780ef2da734e32a8d43a522a580b5e0f3b" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.threeten" name="threetenbp" version="1.6.5">
+         <artifact name="threetenbp-1.6.5.jar">
+            <sha256 value="b2551604c0d6516428e3065213b74461240378a07622201a37ad32638ecb417c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.tmatesoft.sqljet" name="sqljet" version="1.1.10">
          <artifact name="sqljet-1.1.10.jar">
             <sha256 value="df7463424e3560f5e8c8003e1816c0a6ea6e84673921ca5af05b90b0892b3c97" origin="Generated by Gradle"/>

plugins/repository-gcs/build.gradle

@@ -26,35 +26,35 @@ esplugin {
 }
 
 dependencies {
-  api 'com.google.cloud:google-cloud-storage:1.118.1'
-  api 'com.google.cloud:google-cloud-core:2.0.2'
-  api 'com.google.cloud:google-cloud-core-http:2.0.2'
-  runtimeOnly 'com.google.guava:guava:30.1.1-jre'
-  api 'com.google.guava:failureaccess:1.0.1'
+  api 'com.google.cloud:google-cloud-storage:2.13.1'
+  api 'com.google.cloud:google-cloud-core:2.8.28'
+  api 'com.google.cloud:google-cloud-core-http:2.8.28'
+  runtimeOnly 'com.google.guava:guava:31.1-jre'
+  runtimeOnly 'com.google.guava:failureaccess:1.0.1'
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'com.google.api:api-common:2.2.1'
-  api 'com.google.api:gax:2.0.0'
-  api 'org.threeten:threetenbp:1.5.1'
+  api 'com.google.api:api-common:2.3.1'
+  api 'com.google.api:gax:2.20.1'
+  api 'org.threeten:threetenbp:1.6.5'
   api 'com.google.protobuf:protobuf-java-util:3.17.3'
-  api 'com.google.protobuf:protobuf-java:3.21.1'
-  api 'com.google.code.gson:gson:2.8.9'
-  api 'com.google.api.grpc:proto-google-common-protos:2.3.2'
-  api 'com.google.api.grpc:proto-google-iam-v1:1.0.14'
-  api 'com.google.auth:google-auth-library-credentials:1.0.0'
-  api 'com.google.auth:google-auth-library-oauth2-http:1.0.0'
+  api 'com.google.protobuf:protobuf-java:3.21.7'
+  api 'com.google.code.gson:gson:2.10'
+  api 'com.google.api.grpc:proto-google-common-protos:2.9.6'
+  api 'com.google.api.grpc:proto-google-iam-v1:1.6.2'
+  api 'com.google.auth:google-auth-library-credentials:1.11.0'
+  api 'com.google.auth:google-auth-library-oauth2-http:1.11.0'
   api 'com.google.oauth-client:google-oauth-client:1.34.1'
-  api 'com.google.api-client:google-api-client:1.35.1'
-  api 'com.google.http-client:google-http-client:1.39.2'
-  api 'com.google.http-client:google-http-client-gson:1.39.2'
-  api 'com.google.http-client:google-http-client-appengine:1.39.2'
-  api 'com.google.http-client:google-http-client-jackson2:1.39.2'
-  api 'com.google.api:gax-httpjson:0.85.0'
-  api 'io.grpc:grpc-context:1.39.0'
-  api 'io.opencensus:opencensus-api:0.28.0'
-  api 'io.opencensus:opencensus-contrib-http-util:0.28.0'
-  api 'com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1'
+  api 'com.google.api-client:google-api-client:2.1.1'
+  api 'com.google.http-client:google-http-client:1.42.3'
+  api 'com.google.http-client:google-http-client-gson:1.42.3'
+  api 'com.google.http-client:google-http-client-appengine:1.42.3'
+  api 'com.google.http-client:google-http-client-jackson2:1.42.3'
+  api 'com.google.api:gax-httpjson:0.105.1'
+  api 'io.grpc:grpc-context:1.49.2'
+  api 'io.opencensus:opencensus-api:0.31.1'
+  api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
+  api 'com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0'
 
   testImplementation project(':test:fixtures:gcs-fixture')
 }
@@ -177,6 +177,12 @@ tasks.named("thirdPartyAudit").configure {
     'org.apache.http.protocol.HttpProcessor',
     'org.apache.http.protocol.HttpRequestExecutor',
     'javax.jms.Message',
+    // com.google.api.gax optional dependencies
+    'org.graalvm.nativeimage.hosted.Feature',
+    'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess',
+    'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess',
+    'org.graalvm.nativeimage.hosted.Feature$FeatureAccess',
+    'org.graalvm.nativeimage.hosted.RuntimeReflection',
     // commons-logging provided dependencies
     'javax.servlet.ServletContextEvent',
     'javax.servlet.ServletContextListener'

plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java

@@ -14,6 +14,7 @@
 import com.google.api.gax.retrying.RetrySettings;
 import com.google.cloud.http.HttpTransportOptions;
 import com.google.cloud.storage.StorageOptions;
+import com.google.cloud.storage.StorageRetryStrategy;
 import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
@@ -242,6 +243,7 @@ StorageOptions createStorageOptions(
                 ) {
                     StorageOptions options = super.createStorageOptions(gcsClientSettings, httpTransportOptions);
                     return options.toBuilder()
+                        .setStorageRetryStrategy(StorageRetryStrategy.getLegacyStorageRetryStrategy())
                         .setHost(options.getHost())
                         .setCredentials(options.getCredentials())
                         .setRetrySettings(

plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java

@@ -19,6 +19,7 @@
 import com.google.cloud.http.HttpTransportOptions;
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
+import com.google.cloud.storage.StorageRetryStrategy;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -172,6 +173,7 @@ StorageOptions createStorageOptions(
         final HttpTransportOptions httpTransportOptions
     ) {
         final StorageOptions.Builder storageOptionsBuilder = StorageOptions.newBuilder()
+            .setStorageRetryStrategy(StorageRetryStrategy.getLegacyStorageRetryStrategy())
             .setTransportOptions(httpTransportOptions)
             .setHeaderProvider(() -> {
                 final MapBuilder<String, String> mapBuilder = MapBuilder.newMapBuilder();