Upgrade GCS Repository Dependencies (elastic#43142)

* Upgrade to latest GCS SDK and transitive dependencies (I chose the later version here on conflict)
* Remove now unnecessary hack for custom endpoints (the linked bugs were both resolved in the SDK)

plugins/repository-gcs/build.gradle

@@ -25,47 +25,42 @@ esplugin {
 }
 
 dependencies {
-  compile 'com.google.cloud:google-cloud-storage:1.59.0'
-  compile 'com.google.cloud:google-cloud-core:1.59.0'
-  compile 'com.google.guava:guava:20.0'
-  compile "joda-time:joda-time:${versions.joda}"
-  compile 'com.google.http-client:google-http-client:1.24.1'
+  compile 'com.google.cloud:google-cloud-storage:1.77.0'
+  compile 'com.google.cloud:google-cloud-core:1.77.0'
+  compile 'com.google.guava:guava:26.0-jre'
+  compile 'com.google.http-client:google-http-client:1.30.1'
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
-  compile 'com.google.api:api-common:1.7.0'
-  compile 'com.google.api:gax:1.30.0'
+  compile 'com.google.api:api-common:1.8.1'
+  compile 'com.google.api:gax:1.45.0'
   compile 'org.threeten:threetenbp:1.3.3'
-  compile 'com.google.protobuf:protobuf-java-util:3.6.0'
-  compile 'com.google.protobuf:protobuf-java:3.6.0'
+  compile 'com.google.protobuf:protobuf-java-util:3.7.1'
+  compile 'com.google.protobuf:protobuf-java:3.7.1'
   compile 'com.google.code.gson:gson:2.7'
-  compile 'com.google.api.grpc:proto-google-common-protos:1.12.0'
+  compile 'com.google.api.grpc:proto-google-common-protos:1.16.0'
   compile 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
-  compile 'com.google.cloud:google-cloud-core-http:1.59.0'
-  compile 'com.google.auth:google-auth-library-credentials:0.10.0'
-  compile 'com.google.auth:google-auth-library-oauth2-http:0.10.0'
-  compile 'com.google.oauth-client:google-oauth-client:1.24.1'
-  compile 'com.google.api-client:google-api-client:1.24.1'
-  compile 'com.google.http-client:google-http-client-appengine:1.24.1'
-  compile 'com.google.http-client:google-http-client-jackson:1.24.1'
-  compile 'org.codehaus.jackson:jackson-core-asl:1.9.11'
-  compile 'com.google.http-client:google-http-client-jackson2:1.24.1'
-  compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  compile 'com.google.api:gax-httpjson:0.47.0'
-  compile 'io.opencensus:opencensus-api:0.15.0'
+  compile 'com.google.cloud:google-cloud-core-http:1.77.0'
+  compile 'com.google.auth:google-auth-library-credentials:0.16.1'
+  compile 'com.google.auth:google-auth-library-oauth2-http:0.16.1'
+  compile 'com.google.oauth-client:google-oauth-client:1.28.0'
+  compile 'com.google.api-client:google-api-client:1.28.0'
+  compile 'com.google.http-client:google-http-client-appengine:1.29.2'
+  compile 'com.google.http-client:google-http-client-jackson2:1.29.2'
+  compile 'com.google.api:gax-httpjson:0.62.0'
   compile 'io.grpc:grpc-context:1.12.0'
-  compile 'io.opencensus:opencensus-contrib-http-util:0.15.0'
-  compile 'com.google.apis:google-api-services-storage:v1-rev135-1.24.1'
+  compile 'io.opencensus:opencensus-api:0.18.0'
+  compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
+  compile 'com.google.apis:google-api-services-storage:v1-rev20190426-1.28.0'
 }
 
 dependencyLicenses {
   mapping from: /google-cloud-.*/, to: 'google-cloud'
   mapping from: /google-auth-.*/, to: 'google-auth'
   mapping from: /google-http-.*/, to: 'google-http'
   mapping from: /opencensus.*/, to: 'opencensus'
-  mapping from: /jackson-.*/, to: 'jackson'
   mapping from: /http.*/, to: 'httpclient'
   mapping from: /protobuf.*/, to: 'protobuf'
   mapping from: /proto-google.*/, to: 'proto-google'
@@ -81,6 +76,10 @@ thirdPartyAudit {
         'com.google.common.cache.Striped64',
         'com.google.common.cache.Striped64$1',
         'com.google.common.cache.Striped64$Cell',
+        'com.google.common.hash.Striped64',
+        'com.google.common.hash.Striped64$1',
+        'com.google.common.hash.Striped64$Cell',
+        'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
         'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
         'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
         'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',

plugins/repository-gcs/licenses/api-common-1.8.1.jar.sha1

@@ -0,0 +1 @@
+e89befb19b08ad84b262b2f226ab79aefcaa9d7f

plugins/repository-gcs/licenses/gax-1.45.0.jar.sha1

@@ -0,0 +1 @@
+2ade3e3502f9d14e3731347a82ea02372094211f

plugins/repository-gcs/licenses/gax-httpjson-0.62.0.jar.sha1

@@ -0,0 +1 @@
+05a1a4736acd1c4f30304be953532be6aecdc2c9

plugins/repository-gcs/licenses/google-api-client-1.28.0.jar.sha1

@@ -0,0 +1 @@
+8fe155d766ed22480939e3a9db428151e0264d9e

plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20190426-1.28.0.jar.sha1

@@ -0,0 +1 @@
+34dd008901f382507a572f5242d0e5c5ea4ad713

plugins/repository-gcs/licenses/google-auth-library-credentials-0.16.1.jar.sha1

@@ -0,0 +1 @@
+9a15387cc0438ac3f3e625b6050cf39f4e981e13

plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.16.1.jar.sha1

@@ -0,0 +1 @@
+3407d434678faef3439a7012efa336e751ddc623

plugins/repository-gcs/licenses/google-cloud-core-1.77.0.jar.sha1

@@ -0,0 +1 @@
+7cd83a789fde368a999c1793c6297e7b4e56b2ac

plugins/repository-gcs/licenses/google-cloud-core-http-1.77.0.jar.sha1

@@ -0,0 +1 @@
+e16acbc935a7762ba9b220860ae45c2c67d17d8c

plugins/repository-gcs/licenses/google-cloud-storage-1.77.0.jar.sha1

@@ -0,0 +1 @@
+e368e1a8bbc0d0a4354f4e5eec076f38f6966050

plugins/repository-gcs/licenses/google-http-client-1.30.1.jar.sha1

@@ -0,0 +1 @@
+573aacbda8feb0d43f7056291fbce5496f42a6aa

plugins/repository-gcs/licenses/google-http-client-appengine-1.29.2.jar.sha1

@@ -0,0 +1 @@
+d93f4d1d8c2496d75221e53173e4c503b7096a4d

plugins/repository-gcs/licenses/google-http-client-jackson2-1.29.2.jar.sha1

@@ -0,0 +1 @@
+d67891f5a438e1f339387a09628e0ab0af8b612a

plugins/repository-gcs/licenses/google-oauth-client-1.28.0.jar.sha1

@@ -0,0 +1 @@
+9a9e5d0c33b663d6475c96ce79b2949545a113af

plugins/repository-gcs/licenses/guava-26.0-jre.jar.sha1

@@ -0,0 +1 @@
+6a806eff209f36f635f943e16d97491f00f6bfab

plugins/repository-gcs/licenses/opencensus-api-0.18.0.jar.sha1

@@ -0,0 +1 @@
+b89a8f8dfd1e1e0d68d83c82a855624814b19a6e

plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.18.0.jar.sha1

@@ -0,0 +1 @@
+76a37e4a931d5801a9e25b0c0353e5f37c4d1e8e

plugins/repository-gcs/licenses/proto-google-common-protos-1.16.0.jar.sha1

@@ -0,0 +1 @@
+2c5f022ea3b8e8df6a619c4cd8faf9af86022daa

plugins/repository-gcs/licenses/protobuf-java-3.7.1.jar.sha1

@@ -0,0 +1 @@
+0bce1b6dc9e4531169542ab37a1c8641bcaa8afb

plugins/repository-gcs/licenses/protobuf-java-util-3.7.1.jar.sha1

@@ -0,0 +1 @@
+45dc95896cfad26397675fdabef7b032d6db4bb6

plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java

@@ -21,7 +21,6 @@
 
 import com.google.api.client.googleapis.GoogleUtils;
 import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.javanet.DefaultConnectionFactory;
 import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.auth.oauth2.ServiceAccountCredentials;
 import com.google.cloud.http.HttpTransportOptions;
@@ -37,10 +36,7 @@
 import org.elasticsearch.common.util.LazyInitializable;
 
 import java.io.IOException;
-import java.net.HttpURLConnection;
 import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -104,10 +100,16 @@ public Storage client(final String clientName) throws IOException {
      * @return a new client storage instance that can be used to manage objects
      *         (blobs)
      */
-    private Storage createClient(final String clientName, final GoogleCloudStorageClientSettings clientSettings) throws IOException {
+    private static Storage createClient(String clientName, GoogleCloudStorageClientSettings clientSettings) throws IOException {
         logger.debug(() -> new ParameterizedMessage("creating GCS client with client_name [{}], endpoint [{}]", clientName,
                 clientSettings.getHost()));
-        final HttpTransport httpTransport = SocketAccess.doPrivilegedIOException(() -> createHttpTransport(clientSettings.getHost()));
+        final HttpTransport httpTransport = SocketAccess.doPrivilegedIOException(() -> {
+            final NetHttpTransport.Builder builder = new NetHttpTransport.Builder();
+            // requires java.lang.RuntimePermission "setFactory"
+            // Pin the TLS trust certificates.
+            builder.trustCertificates(GoogleUtils.getCertificateTrustStore());
+            return builder.build();
+        });
         final HttpTransportOptions httpTransportOptions = HttpTransportOptions.newBuilder()
                 .setConnectTimeout(toTimeout(clientSettings.getConnectTimeout()))
                 .setReadTimeout(toTimeout(clientSettings.getReadTimeout()))
@@ -145,54 +147,6 @@ private Storage createClient(final String clientName, final GoogleCloudStorageCl
         return storageOptionsBuilder.build().getService();
     }
 
-    /**
-     * Pins the TLS trust certificates and, more importantly, overrides connection
-     * URLs in the case of a custom endpoint setting because some connections don't
-     * fully honor this setting (bugs in the SDK). The default connection factory
-     * opens a new connection for each request. This is required for the storage
-     * instance to be thread-safe.
-     **/
-    private static HttpTransport createHttpTransport(final String endpoint) throws Exception {
-        final NetHttpTransport.Builder builder = new NetHttpTransport.Builder();
-        // requires java.lang.RuntimePermission "setFactory"
-        builder.trustCertificates(GoogleUtils.getCertificateTrustStore());
-        if (Strings.hasLength(endpoint)) {
-            final URL endpointUrl = URI.create(endpoint).toURL();
-            // it is crucial to open a connection for each URL (see {@code
-            // DefaultConnectionFactory#openConnection}) instead of reusing connections,
-            // because the storage instance has to be thread-safe as it is cached.
-            builder.setConnectionFactory(new DefaultConnectionFactory() {
-                @Override
-                public HttpURLConnection openConnection(final URL originalUrl) throws IOException {
-                    // test if the URL is built correctly, ie following the `host` setting
-                    if (originalUrl.getHost().equals(endpointUrl.getHost()) && originalUrl.getPort() == endpointUrl.getPort()
-                            && originalUrl.getProtocol().equals(endpointUrl.getProtocol())) {
-                        return super.openConnection(originalUrl);
-                    }
-                    // override connection URLs because some don't follow the config. See
-                    // https://github.com/GoogleCloudPlatform/google-cloud-java/issues/3254 and
-                    // https://github.com/GoogleCloudPlatform/google-cloud-java/issues/3255
-                    URI originalUri;
-                    try {
-                        originalUri = originalUrl.toURI();
-                    } catch (final URISyntaxException e) {
-                        throw new RuntimeException(e);
-                    }
-                    String overridePath = "/";
-                    if (originalUri.getRawPath() != null) {
-                        overridePath = originalUri.getRawPath();
-                    }
-                    if (originalUri.getRawQuery() != null) {
-                        overridePath += "?" + originalUri.getRawQuery();
-                    }
-                    return super.openConnection(
-                            new URL(endpointUrl.getProtocol(), endpointUrl.getHost(), endpointUrl.getPort(), overridePath));
-                }
-            });
-        }
-        return builder.build();
-    }
-
     /**
      * Converts timeout values from the settings to a timeout value for the Google
      * Cloud SDK

plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java

@@ -281,13 +281,23 @@ public void close() {
         return null;
     }
 
+    @Override
+    public WriteChannel writer(URL signedURL) {
+        return null;
+    }
+
     // Everything below this line is not implemented.
 
     @Override
     public CopyWriter copy(CopyRequest copyRequest) {
         return null;
     }
 
+    @Override
+    public Blob create(BlobInfo blobInfo, byte[] content, int offset, int length, BlobTargetOption... options) {
+        return null;
+    }
+
     @Override
     public Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) {
         return null;