[DO NOT MERGE][BEAM-14118, #21639] Test gRPC 1.48.1

.test-infra/validate-runner/build.gradle

@@ -22,6 +22,7 @@ group 'org.apache.beam'
 description = "Apache Beam :: Validate :: Runner"
 
 repositories {
+    maven { url "${project.rootDir}/tempLib" }
     mavenCentral()
     maven {
         url "https://repo.jenkins-ci.org/releases/"

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -719,7 +719,7 @@ class BeamModulePlugin implements Plugin<Project> {
         testcontainers_postgresql                   : "org.testcontainers:postgresql:$testcontainers_version",
         testcontainers_mysql                        : "org.testcontainers:mysql:$testcontainers_version",
         testcontainers_gcloud                       : "org.testcontainers:gcloud:$testcontainers_version",
-        vendored_grpc_1_43_2                        : "org.apache.beam:beam-vendor-grpc-1_43_2:0.1",
+        vendored_grpc_1_48_1                        : "org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT",
         vendored_guava_26_0_jre                     : "org.apache.beam:beam-vendor-guava-26_0-jre:0.1",
         vendored_calcite_1_28_0                     : "org.apache.beam:beam-vendor-calcite-1_28_0:0.2",
         woodstox_core_asl                           : "org.codehaus.woodstox:woodstox-core-asl:4.4.1",
@@ -2129,10 +2129,10 @@ class BeamModulePlugin implements Plugin<Project> {
           archivesBaseName: configuration.archivesBaseName,
           automaticModuleName: configuration.automaticModuleName,
           shadowJarValidationExcludes: it.shadowJarValidationExcludes,
-          shadowClosure: GrpcVendoring_1_43_2.shadowClosure() << {
+          shadowClosure: GrpcVendoring_1_48_1.shadowClosure() << {
             // We perform all the code relocations but don't include
             // any of the actual dependencies since they will be supplied
-            // by org.apache.beam:beam-vendor-grpc-v1p43p2
+            // by org.apache.beam:beam-vendor-grpc-v1p48p1
             dependencies {
               include(dependency { return false })
             }
@@ -2149,14 +2149,14 @@ class BeamModulePlugin implements Plugin<Project> {
       project.protobuf {
         protoc {
           // The artifact spec for the Protobuf Compiler
-          artifact = "com.google.protobuf:protoc:${GrpcVendoring_1_43_2.protobuf_version}" }
+          artifact = "com.google.protobuf:protoc:${GrpcVendoring_1_48_1.protobuf_version}" }
 
         // Configure the codegen plugins
         plugins {
           // An artifact spec for a protoc plugin, with "grpc" as
           // the identifier, which can be referred to in the "plugins"
           // container of the "generateProtoTasks" closure.
-          grpc { artifact = "io.grpc:protoc-gen-grpc-java:${GrpcVendoring_1_43_2.grpc_version}" }
+          grpc { artifact = "io.grpc:protoc-gen-grpc-java:${GrpcVendoring_1_48_1.grpc_version}" }
         }
 
         generateProtoTasks {
@@ -2170,7 +2170,7 @@ class BeamModulePlugin implements Plugin<Project> {
         }
       }
 
-      project.dependencies GrpcVendoring_1_43_2.dependenciesClosure() << { shadow project.ext.library.java.vendored_grpc_1_43_2 }
+      project.dependencies GrpcVendoring_1_48_1.dependenciesClosure() << { shadow project.ext.library.java.vendored_grpc_1_48_1 }
     }
 
     /** ***********************************************************************************************/

buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_48_1.groovy

@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.gradle
+
+/**
+ * Utilities for working with our vendored version of gRPC.
+ * 
+ * To update:
+ * 1. Determine the set of io.grpc libraries we want to include, most likely a superset of
+ *    of the previous vendored gRPC version.
+ * 2. Use mvn dependency:tree and https://search.maven.org/search?q=g:io.grpc
+ *    to determine dependency tree. You may need to search for optional dependencies
+ *    and determine if they are needed (e.g. conscrypt).
+ * 3. Validate built artifacts by running linkage tool
+ *    (https://github.com/apache/beam/tree/master/vendor#how-to-validate-the-vendored-dependencies)
+ *    and unit and integration tests in a PR (e.g. https://github.com/apache/beam/pull/16460,
+ *    https://github.com/apache/beam/pull/16459)
+ */
+class GrpcVendoring_1_48_1 {
+  static def grpc_version = "1.48.1"
+
+  // See https://github.com/grpc/grpc-java/blob/v1.48.1/gradle/libs.versions.toml
+  // or https://search.maven.org/search?q=g:io.grpc%201.48.1
+  static def guava_version = "31.1-jre"
+  static def protobuf_version = "3.21.1"
+  static def gson_version = "2.9.0"
+  static def google_auth_version = "1.4.0"
+  static def opencensus_version = "0.31.0"
+  static def conscrypt_version = "2.5.2"
+  static def proto_google_common_protos_version = "2.9.0"
+  static def netty_version = "4.1.77.Final"
+  static def netty_tcnative_version = "2.0.53.Final"
+
+  /** Returns the list of implementation time dependencies. */
+  static List<String> dependencies() {
+    return [
+      "com.google.guava:guava:$guava_version",
+      "com.google.protobuf:protobuf-java:$protobuf_version",
+      "com.google.protobuf:protobuf-java-util:$protobuf_version",
+      "com.google.code.gson:gson:$gson_version",
+      "io.grpc:grpc-auth:$grpc_version",
+      "io.grpc:grpc-core:$grpc_version",
+      "io.grpc:grpc-context:$grpc_version",
+      "io.grpc:grpc-netty:$grpc_version",
+      "io.grpc:grpc-protobuf:$grpc_version",
+      "io.grpc:grpc-stub:$grpc_version",
+      "io.grpc:grpc-testing:$grpc_version",
+      // Use a classifier to ensure we get the jar containing native libraries. In the future
+      // hopefully netty releases a single jar containing native libraries for all architectures.
+      "io.netty:netty-transport-native-epoll:$netty_version:linux-x86_64",
+      "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version",
+      "com.google.auth:google-auth-library-credentials:$google_auth_version",
+      "com.google.api.grpc:proto-google-common-protos:$proto_google_common_protos_version",
+      "io.opencensus:opencensus-api:$opencensus_version",
+      "io.opencensus:opencensus-contrib-grpc-metrics:$opencensus_version",
+    ]
+  }
+
+  /**
+   * Returns the list of runtime time dependencies that should be exported as runtime
+   * dependencies within the vendored jar.
+   */
+  static List<String> runtimeDependencies() {
+    return [
+      'com.google.errorprone:error_prone_annotations:2.14.0',
+      // TODO(BEAM-9288): Enable relocation for conscrypt
+      "org.conscrypt:conscrypt-openjdk-uber:$conscrypt_version"
+    ]
+  }
+
+  /**
+   * Returns the list of test dependencies.
+   */
+  static List<String> testDependencies() {
+    return [
+      'junit:junit:4.12',
+    ]
+  }
+
+  static Map<String, String> relocations() {
+    // The relocation paths below specifically use gRPC and the full version string as
+    // the code relocation prefix. See https://lists.apache.org/thread.html/4c12db35b40a6d56e170cd6fc8bb0ac4c43a99aa3cb7dbae54176815@%3Cdev.beam.apache.org%3E
+    // for further details.
+
+    // To produce the list of necessary relocations, one needs to start with a set of target
+    // packages that one wants to vendor, find all necessary transitive dependencies of that
+    // set and provide relocations for each such that all necessary packages and their
+    // dependencies are relocated. Any optional dependency that doesn't need relocation
+    // must be excluded via an 'exclude' rule. There is additional complexity of libraries that use
+    // JNI or reflection and have to be handled on case by case basis by learning whether
+    // they support relocation and how would one go about doing it by reading any documentation
+    // those libraries may provide. The 'validateShadedJarDoesntLeakNonOrgApacheBeamClasses'
+    // ensures that there are no classes outside of the 'org.apache.beam' namespace.
+
+    String version = "v1p48p1";
+    String prefix = "org.apache.beam.vendor.grpc.${version}";
+    List<String> packagesToRelocate = [
+      // guava uses the com.google.common and com.google.thirdparty package namespaces
+      "com.google.common",
+      "com.google.thirdparty",
+      "com.google.protobuf",
+      "com.google.gson",
+      "com.google.auth",
+      "com.google.api",
+      "com.google.cloud",
+      "com.google.logging",
+      "com.google.longrunning",
+      "com.google.rpc",
+      "com.google.type",
+      "com.google.geo.type",
+      "io.grpc",
+      "io.netty",
+      "io.opencensus",
+      "io.perfmark",
+    ]
+
+    return packagesToRelocate.collectEntries {
+      [ (it): "${prefix}.${it}" ]
+    } + [
+      // Adapted from https://github.com/grpc/grpc-java/blob/e283f70ad91f99c7fee8b31b605ef12a4f9b1690/netty/shaded/build.gradle#L41
+      // We       "io.netty": "${prefix}.io.netty",have to be careful with these replacements as they must not match any
+      // string in NativeLibraryLoader, else they cause corruption. Note that
+      // this includes concatenation of string literals and constants.
+      'META-INF/native/libnetty': "META-INF/native/liborg_apache_beam_vendor_grpc_${version}_netty",
+      'META-INF/native/netty': "META-INF/native/org_apache_beam_vendor_grpc_${version}_netty",
+      'META-INF/native/lib-netty': "META-INF/native/lib-org-apache-beam-vendor-grpc-${version}-netty",
+    ]
+  }
+
+  /** Returns the list of shading exclusions. */
+  static List<String> exclusions() {
+    return [
+      // Don't include in the vendored jar:
+      // android annotations, errorprone, checkerframework, JDK8 annotations, objenesis, junit,
+      // commons-logging, log4j, slf4j and mockito
+      "android/annotation/**/",
+      "com/google/errorprone/**",
+      "com/google/instrumentation/**",
+      "com/google/j2objc/annotations/**",
+      "io/netty/handler/codec/marshalling/**",
+      "io/netty/handler/codec/spdy/**",
+      "io/netty/handler/codec/compression/JZlib*",
+      "io/netty/handler/codec/compression/Lz4*",
+      "io/netty/handler/codec/compression/Lzf*",
+      "io/netty/handler/codec/compression/Lzma*",
+      "io/netty/handler/codec/protobuf/Protobuf*Nano.class",
+      "io/netty/util/internal/logging/CommonsLogger*",
+      "io/netty/util/internal/logging/LocationAwareSlf4JLogger*",
+      "io/netty/util/internal/logging/Log4JLogger*",
+      "io/netty/util/internal/logging/Log4J2Logger*",
+      "javax/annotation/**",
+      "junit/**",
+      "module-info.class",
+      "org/checkerframework/**",
+      "org/codehaus/mojo/animal_sniffer/**",
+      "org/conscrypt/**",
+      "META-INF/native/libconscrypt**",
+      "META-INF/native/conscrypt**",
+      "org/hamcrest/**",
+      "org/junit/**",
+      "org/mockito/**",
+      "org/objenesis/**",
+    ]
+  }
+
+  /**
+   * Returns a closure contaning the dependencies map used for shading gRPC within the main
+   * Apache Beam project.
+   */
+  static Object dependenciesClosure() {
+    return {
+      dependencies().each { implementation it }
+      runtimeDependencies().each { shadow it }
+    }
+  }
+
+  /**
+   * Returns a closure with the code relocation configuration for shading gRPC within the main
+   * Apache Beam project.
+   */
+  static Object shadowClosure() {
+    return {
+      relocations().each { srcNamespace, destNamespace ->
+        relocate srcNamespace, destNamespace
+      }
+      exclusions().each { exclude it }
+    }
+  }
+}

buildSrc/src/main/groovy/org/apache/beam/gradle/Repositories.groovy

@@ -25,6 +25,7 @@ class Repositories {
   static void register(Project project) {
 
     project.repositories {
+      maven { url "${project.rootDir}/tempLib" }
       maven { url project.offlineRepositoryRoot }
 
       // To run gradle in offline mode, one must first invoke

examples/java/build.gradle

@@ -80,7 +80,7 @@ dependencies {
   implementation library.java.commons_io
   implementation library.java.commons_csv
   runtimeOnly project(path: ":runners:direct-java", configuration: "shadow")
-  implementation library.java.vendored_grpc_1_43_2
+  implementation library.java.vendored_grpc_1_48_1
   implementation library.java.vendored_guava_26_0_jre
   implementation "com.google.api.grpc:proto-google-cloud-language-v1:1.81.4"
   implementation ("io.confluent:kafka-avro-serializer:5.3.2") {

examples/java/src/main/java/org/apache/beam/examples/complete/datatokenization/transforms/DataProtectors.java

@@ -48,9 +48,9 @@
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.gson.Gson;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.gson.JsonArray;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.gson.JsonObject;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.gson.Gson;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.gson.JsonArray;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.gson.JsonObject;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Throwables;
 import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpEntity;

examples/java/src/main/java/org/apache/beam/examples/complete/kafkatopubsub/kafka/consumer/Utils.java

@@ -28,8 +28,8 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 import org.apache.beam.examples.complete.kafkatopubsub.options.KafkaToPubsubOptions;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.gson.JsonObject;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.gson.JsonParser;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.gson.JsonObject;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.gson.JsonParser;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;

runners/core-construction-java/build.gradle

@@ -54,7 +54,7 @@ dependencies {
   implementation project(path: ":model:job-management", configuration: "shadow")
   implementation project(path: ":sdks:java:core", configuration: "shadow")
   implementation project(path: ":sdks:java:fn-execution")
-  implementation library.java.vendored_grpc_1_43_2
+  implementation library.java.vendored_grpc_1_48_1
   implementation library.java.vendored_guava_26_0_jre
   implementation library.java.classgraph
   implementation library.java.jackson_core

runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/BeamUrns.java

@@ -18,7 +18,7 @@
 package org.apache.beam.runners.core.construction;
 
 import org.apache.beam.model.pipeline.v1.RunnerApi;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ProtocolMessageEnum;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.ProtocolMessageEnum;
 
 /** Returns the standard URN of a given enum annotated with [(standard_urn)]. */
 public class BeamUrns {

runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslation.java

@@ -28,7 +28,7 @@
 import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.util.SerializableUtils;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.ByteString;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.BiMap;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableBiMap;

runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CoderTranslators.java

@@ -37,7 +37,7 @@
 import org.apache.beam.sdk.util.ShardedKey;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.InvalidProtocolBufferException;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
 
 /** {@link CoderTranslator} implementations for known coder types. */

runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CombineTranslation.java

@@ -39,7 +39,7 @@
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.ByteString;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;

runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java

@@ -34,7 +34,7 @@
 import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.ByteString;
 
 /**
  * Utility methods for translating a {@link View} transforms to and from {@link RunnerApi}

runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DefaultExpansionServiceClientFactory.java

@@ -23,7 +23,7 @@
 import org.apache.beam.model.expansion.v1.ExpansionApi;
 import org.apache.beam.model.expansion.v1.ExpansionServiceGrpc;
 import org.apache.beam.model.pipeline.v1.Endpoints;
-import org.apache.beam.vendor.grpc.v1p43p2.io.grpc.ManagedChannel;
+import org.apache.beam.vendor.grpc.v1p48p1.io.grpc.ManagedChannel;
 
 /** Default factory for ExpansionServiceClient used by External transform. */
 public class DefaultExpansionServiceClientFactory implements ExpansionServiceClientFactory {

runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/DisplayDataTranslation.java

@@ -25,7 +25,7 @@
 import org.apache.beam.model.pipeline.v1.RunnerApi;
 import org.apache.beam.model.pipeline.v1.RunnerApi.StandardDisplayData;
 import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.ByteString;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
 

runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java

@@ -47,8 +47,8 @@
 import org.apache.beam.sdk.util.ReleaseInfo;
 import org.apache.beam.sdk.util.ZipFiles;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.InvalidProtocolBufferException;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings;

runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/External.java

@@ -52,9 +52,9 @@
 import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.PValues;
 import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
-import org.apache.beam.vendor.grpc.v1p43p2.io.grpc.ManagedChannel;
-import org.apache.beam.vendor.grpc.v1p43p2.io.grpc.ManagedChannelBuilder;
+import org.apache.beam.vendor.grpc.v1p48p1.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p48p1.io.grpc.ManagedChannel;
+import org.apache.beam.vendor.grpc.v1p48p1.io.grpc.ManagedChannelBuilder;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;