Skip to content

Commit

Permalink
[BEAM-14118, #21639] Vendor gRPC 1.48.1 (#22607)
Browse files Browse the repository at this point in the history 
Linkage checker failures under 1.48.1:

Class org.eclipse.jetty.npn.NextProtoNego is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.JettyNpnSslEngine (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.eclipse.jetty.alpn.ALPN$ClientProvider is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.JettyAlpnSslEngine (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.eclipse.jetty.alpn.ALPN is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.JettyAlpnSslEngine (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.asn1.x500.X500Name is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.operator.jcajce.JcaContentSignerBuilder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.cert.X509v3CertificateBuilder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.cert.jcajce.JcaX509CertificateConverter is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.jce.provider.BouncyCastleProvider is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.eclipse.jetty.npn.NextProtoNego$ClientProvider is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.JettyNpnSslEngine (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.eclipse.jetty.npn.NextProtoNego$ServerProvider is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.JettyNpnSslEngine (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.eclipse.jetty.alpn.ALPN$ServerProvider is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.ssl.JettyAlpnSslEngine (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.jcraft.jzlib.JZlib$WrapperType is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.ZlibUtil (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.jcraft.jzlib.JZlib is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.ZlibUtil (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.jcraft.jzlib.Inflater is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.ZlibUtil (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.jcraft.jzlib.Deflater is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.ZlibUtil (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.aayushatharva.brotli4j.decoder.DecoderJNI$Status is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.BrotliDecoder (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.github.luben.zstd.Zstd is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.ZstdEncoder (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.aayushatharva.brotli4j.encoder.Encoder$Parameters is not found;
  referenced by 3 class files
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.StandardCompressionOptions (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.BrotliOptions (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.BrotliEncoder (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.BrotliDecoder (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.aayushatharva.brotli4j.encoder.Encoder$Mode is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.BrotliOptions (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.JZlibEncoder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.ZlibCodecFactory (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.JZlibDecoder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.ZlibCodecFactory (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.aayushatharva.brotli4j.encoder.Encoders is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.handler.codec.compression.BrotliEncoder (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.slf4j.Logger is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.util.internal.logging.Slf4JLogger (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.slf4j.LoggerFactory is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.util.internal.logging.Slf4JLoggerFactory (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.slf4j.helpers.NOPLoggerFactory is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.util.internal.logging.Slf4JLoggerFactory (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.slf4j.spi.LocationAwareLogger is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.util.internal.logging.Slf4JLoggerFactory (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.apache.beam.vendor.grpc.v1p48p1.io.netty.util.internal.logging.LocationAwareSlf4JLogger is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.util.internal.logging.Slf4JLoggerFactory (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class reactor.blockhound.integration.BlockHoundIntegration is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.util.internal.Hidden (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Class reactor.blockhound.BlockHound$Builder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p48p1.io.netty.util.internal.Hidden (org.apache.beam:beam-vendor-grpc-1_48_1:0.1-SNAPSHOT)
  Cause:
    Unknown
Linkage checker failures under 1.43.2:

Class org.eclipse.jetty.npn.NextProtoNego$ServerProvider is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.JettyNpnSslEngine (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.asn1.x500.X500Name is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.operator.jcajce.JcaContentSignerBuilder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.cert.X509v3CertificateBuilder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.cert.jcajce.JcaX509CertificateConverter is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.bouncycastle.jce.provider.BouncyCastleProvider is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.eclipse.jetty.alpn.ALPN is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.JettyAlpnSslEngine (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.eclipse.jetty.npn.NextProtoNego is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.JettyNpnSslEngine (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.eclipse.jetty.alpn.ALPN$ClientProvider is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.JettyAlpnSslEngine (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.eclipse.jetty.alpn.ALPN$ServerProvider is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.ssl.JettyAlpnSslEngine (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.codec.compression.JZlibEncoder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.codec.compression.ZlibCodecFactory (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.codec.compression.JZlibDecoder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.codec.compression.ZlibCodecFactory (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.jcraft.jzlib.JZlib$WrapperType is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.codec.compression.ZlibUtil (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.jcraft.jzlib.JZlib is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.codec.compression.ZlibUtil (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.jcraft.jzlib.Inflater is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.codec.compression.ZlibUtil (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class com.jcraft.jzlib.Deflater is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.handler.codec.compression.ZlibUtil (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class reactor.blockhound.integration.BlockHoundIntegration is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.util.internal.Hidden (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class reactor.blockhound.BlockHound$Builder is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.util.internal.Hidden (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.slf4j.Logger is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.util.internal.logging.Slf4JLogger (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.slf4j.LoggerFactory is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.util.internal.logging.Slf4JLoggerFactory (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.slf4j.helpers.NOPLoggerFactory is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.util.internal.logging.Slf4JLoggerFactory (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.slf4j.spi.LocationAwareLogger is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.util.internal.logging.Slf4JLoggerFactory (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown
Class org.apache.beam.vendor.grpc.v1p43p2.io.netty.util.internal.logging.LocationAwareSlf4JLogger is not found;
  referenced by 1 class file
    org.apache.beam.vendor.grpc.v1p43p2.io.netty.util.internal.logging.Slf4JLoggerFactory (org.apache.beam:beam-vendor-grpc-1_43_2:0.1-SNAPSHOT)
  Cause:
    Unknown

The differences are in optional new compression algorithms zstd and brotli and NextProtoNego$ClientProvider which are safe to exclude.
  • Loading branch information
@lukecwik
lukecwik committed Aug 5, 2022
1 parent d0e523b commit db8db0b
Show file tree
Hide file tree
Showing 4 changed files with 254 additions and 8 deletions.
205 changes: 205 additions & 0 deletions buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_48_1.groovy
View file
@@ -0,0 +1,205 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.beam.gradle

/**
* Utilities for working with our vendored version of gRPC.
*
* To update:
* 1. Determine the set of io.grpc libraries we want to include, most likely a superset of
* of the previous vendored gRPC version.
* 2. Use mvn dependency:tree and https://search.maven.org/search?q=g:io.grpc
* to determine dependency tree. You may need to search for optional dependencies
* and determine if they are needed (e.g. conscrypt).
* 3. Validate built artifacts by running linkage tool
* (https://github.com/apache/beam/tree/master/vendor#how-to-validate-the-vendored-dependencies)
* and unit and integration tests in a PR (e.g. https://github.com/apache/beam/pull/16460,
* https://github.com/apache/beam/pull/16459)
*/
class GrpcVendoring_1_48_1 {
static def grpc_version = "1.48.1"

// See https://github.com/grpc/grpc-java/blob/v1.48.1/gradle/libs.versions.toml
// or https://search.maven.org/search?q=g:io.grpc%201.48.1
static def guava_version = "31.1-jre"
static def protobuf_version = "3.21.1"
static def gson_version = "2.9.0"
static def google_auth_version = "1.4.0"
static def opencensus_version = "0.31.0"
static def conscrypt_version = "2.5.2"
static def proto_google_common_protos_version = "2.9.0"
static def netty_version = "4.1.77.Final"
static def netty_tcnative_version = "2.0.53.Final"

/** Returns the list of implementation time dependencies. */
static List<String> dependencies() {
return [
"com.google.guava:guava:$guava_version",
"com.google.protobuf:protobuf-java:$protobuf_version",
"com.google.protobuf:protobuf-java-util:$protobuf_version",
"com.google.code.gson:gson:$gson_version",
"io.grpc:grpc-auth:$grpc_version",
"io.grpc:grpc-core:$grpc_version",
"io.grpc:grpc-context:$grpc_version",
"io.grpc:grpc-netty:$grpc_version",
"io.grpc:grpc-protobuf:$grpc_version",
"io.grpc:grpc-stub:$grpc_version",
"io.grpc:grpc-testing:$grpc_version",
// Use a classifier to ensure we get the jar containing native libraries. In the future
// hopefully netty releases a single jar containing native libraries for all architectures.
"io.netty:netty-transport-native-epoll:$netty_version:linux-x86_64",
"io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version",
"com.google.auth:google-auth-library-credentials:$google_auth_version",
"com.google.api.grpc:proto-google-common-protos:$proto_google_common_protos_version",
"io.opencensus:opencensus-api:$opencensus_version",
"io.opencensus:opencensus-contrib-grpc-metrics:$opencensus_version",
]
}

/**
* Returns the list of runtime time dependencies that should be exported as runtime
* dependencies within the vendored jar.
*/
static List<String> runtimeDependencies() {
return [
'com.google.errorprone:error_prone_annotations:2.14.0',
// TODO(BEAM-9288): Enable relocation for conscrypt
"org.conscrypt:conscrypt-openjdk-uber:$conscrypt_version"
]
}

/**
* Returns the list of test dependencies.
*/
static List<String> testDependencies() {
return [
'junit:junit:4.12',
]
}

static Map<String, String> relocations() {
// The relocation paths below specifically use gRPC and the full version string as
// the code relocation prefix. See https://lists.apache.org/thread.html/4c12db35b40a6d56e170cd6fc8bb0ac4c43a99aa3cb7dbae54176815@%3Cdev.beam.apache.org%3E
// for further details.

// To produce the list of necessary relocations, one needs to start with a set of target
// packages that one wants to vendor, find all necessary transitive dependencies of that
// set and provide relocations for each such that all necessary packages and their
// dependencies are relocated. Any optional dependency that doesn't need relocation
// must be excluded via an 'exclude' rule. There is additional complexity of libraries that use
// JNI or reflection and have to be handled on case by case basis by learning whether
// they support relocation and how would one go about doing it by reading any documentation
// those libraries may provide. The 'validateShadedJarDoesntLeakNonOrgApacheBeamClasses'
// ensures that there are no classes outside of the 'org.apache.beam' namespace.

String version = "v1p48p1";
String prefix = "org.apache.beam.vendor.grpc.${version}";
List<String> packagesToRelocate = [
// guava uses the com.google.common and com.google.thirdparty package namespaces
"com.google.common",
"com.google.thirdparty",
"com.google.protobuf",
"com.google.gson",
"com.google.auth",
"com.google.api",
"com.google.cloud",
"com.google.logging",
"com.google.longrunning",
"com.google.rpc",
"com.google.type",
"com.google.geo.type",
"io.grpc",
"io.netty",
"io.opencensus",
"io.perfmark",
]

return packagesToRelocate.collectEntries {
[ (it): "${prefix}.${it}" ]
} + [
// Adapted from https://github.com/grpc/grpc-java/blob/e283f70ad91f99c7fee8b31b605ef12a4f9b1690/netty/shaded/build.gradle#L41
// We "io.netty": "${prefix}.io.netty",have to be careful with these replacements as they must not match any
// string in NativeLibraryLoader, else they cause corruption. Note that
// this includes concatenation of string literals and constants.
'META-INF/native/libnetty': "META-INF/native/liborg_apache_beam_vendor_grpc_${version}_netty",
'META-INF/native/netty': "META-INF/native/org_apache_beam_vendor_grpc_${version}_netty",
'META-INF/native/lib-netty': "META-INF/native/lib-org-apache-beam-vendor-grpc-${version}-netty",
]
}

/** Returns the list of shading exclusions. */
static List<String> exclusions() {
return [
// Don't include in the vendored jar:
// android annotations, errorprone, checkerframework, JDK8 annotations, objenesis, junit,
// commons-logging, log4j, slf4j and mockito
"android/annotation/**/",
"com/google/errorprone/**",
"com/google/instrumentation/**",
"com/google/j2objc/annotations/**",
"io/netty/handler/codec/marshalling/**",
"io/netty/handler/codec/spdy/**",
"io/netty/handler/codec/compression/JZlib*",
"io/netty/handler/codec/compression/Lz4*",
"io/netty/handler/codec/compression/Lzf*",
"io/netty/handler/codec/compression/Lzma*",
"io/netty/handler/codec/protobuf/Protobuf*Nano.class",
"io/netty/util/internal/logging/CommonsLogger*",
"io/netty/util/internal/logging/LocationAwareSlf4JLogger*",
"io/netty/util/internal/logging/Log4JLogger*",
"io/netty/util/internal/logging/Log4J2Logger*",
"javax/annotation/**",
"junit/**",
"module-info.class",
"org/checkerframework/**",
"org/codehaus/mojo/animal_sniffer/**",
"org/conscrypt/**",
"META-INF/native/libconscrypt**",
"META-INF/native/conscrypt**",
"org/hamcrest/**",
"org/junit/**",
"org/mockito/**",
"org/objenesis/**",
]
}

/**
* Returns a closure contaning the dependencies map used for shading gRPC within the main
* Apache Beam project.
*/
static Object dependenciesClosure() {
return {
dependencies().each { implementation it }
runtimeDependencies().each { shadow it }
}
}

/**
* Returns a closure with the code relocation configuration for shading gRPC within the main
* Apache Beam project.
*/
static Object shadowClosure() {
return {
relocations().each { srcNamespace, destNamespace ->
relocate srcNamespace, destNamespace
}
exclusions().each { exclude it }
}
}
}
1 change: 1 addition & 0 deletions settings.gradle.kts
View file
Expand Up @@ -238,6 +238,7 @@ include(":sdks:python:test-suites:tox:py37")
include(":sdks:python:test-suites:tox:py38")
include(":sdks:python:test-suites:tox:py39")
include(":vendor:grpc-1_43_2")
include(":vendor:grpc-1_48_1")
include(":vendor:calcite-1_28_0")
include(":vendor:guava-26_0-jre")
include(":website")
Expand Down
19 changes: 11 additions & 8 deletions vendor/README.md
View file
Expand Up @@ -32,12 +32,12 @@ The upgrading of the vendored dependencies should be performed in two steps:
The [linkage tool](https://lists.apache.org/thread.html/eb5d95b9a33d7e32dc9bcd0f7d48ba8711d42bd7ed03b9cf0f1103f1%40%3Cdev.beam.apache.org%3E)
is useful for the vendored dependency upgrades. It reports the linkage errors across multiple Apache Beam artifact ids.

For example, when we upgrade the version of gRPC to 1.43.2 and the version of the vendored gRPC is 0.1-SNAPSHOT,
For example, when we upgrade the version of gRPC to 1.48.1 and the version of the vendored gRPC is 0.1-SNAPSHOT,
we could run the linkage tool as following:

```
$ ./gradlew -p vendor/grpc-1_43_2 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly
$ ./gradlew -PvendoredDependenciesOnly -Ppublishing -PjavaLinkageArtifactIds=beam-vendor-grpc-1_43_2:0.1-SNAPSHOT :checkJavaLinkage
$ ./gradlew -p vendor/grpc-1_48_1 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly
$ ./gradlew -PvendoredDependenciesOnly -Ppublishing -PjavaLinkageArtifactIds=beam-vendor-grpc-1_48_1:0.1-SNAPSHOT :checkJavaLinkage
```

### Known Linkage Errors in the Vendored gRPC Dependencies
Expand All @@ -62,7 +62,7 @@ references to the missing classes. Here are the known linkage errors:
The `io.netty.handler.ssl` package has classes that have references to missing classes in other
unused optional SSL implementations.
- References from `io.netty.handler.codec.compression`: Beam does not use the optional dependencies
for compression algorithms (jzlib, lzma, and lzf) through Netty's features.
for compression algorithms (brotli, jzlib, lzma, lzf, and zstd) through Netty's features.
- References to `com.google.protobuf.nano` and `org.jboss.marshalling`: Beam does not use the
optional serialization algorithms.
- References from `io.netty.util.internal.logging`: Netty's logging framework can choose available
Expand All @@ -77,19 +77,22 @@ Once you've verified using the linkage tool, you can test new artifacts by runni

Example PRs:
- Updating gRPC version (large) https://github.com/apache/beam/pull/16460
- Testing updated gRPC version (large) https://github.com/apache/beam/pull/22595
- Updating protobuf for calcite (minor version update): https://github.com/apache/beam/pull/16476

Steps:

1. Generate new artifact files with `publishMavenJavaPublicationToMavenLocal` and
copy to a folder in Beam (e.g. `tempLib`):
copy to the `tempLib` folder in Beam:

```
./gradlew -p vendor/grpc-1_43_2 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly
./gradlew -p vendor/grpc-1_48_1 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly
mkdir -p tempLib/org/apache/beam
# Copy files (jar/poms/metadata) to your beam repository
cp -R ~/.m2/repository/org/apache/beam/beam-vendor-grpc-1_43_2/ \
$BEAMDIR/tempLib/org/apache/beam/beam-vendor-grpc-1_43_2
cp -R ~/.m2/repository/org/apache/beam/beam-vendor-grpc-1_48_1/ \
tempLib/org/apache/beam
```

2. Add the folder to the expected project repositories:
Expand Down
37 changes: 37 additions & 0 deletions vendor/grpc-1_48_1/build.gradle
View file
@@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import org.apache.beam.gradle.GrpcVendoring_1_48_1

plugins { id 'org.apache.beam.vendor-java' }

description = "Apache Beam :: Vendored Dependencies :: gRPC :: 1.48.1"

group = "org.apache.beam"
version = "0.1"

vendorJava(
dependencies: GrpcVendoring_1_48_1.dependencies(),
runtimeDependencies: GrpcVendoring_1_48_1.runtimeDependencies(),
testDependencies: GrpcVendoring_1_48_1.testDependencies(),
relocations: GrpcVendoring_1_48_1.relocations(),
exclusions: GrpcVendoring_1_48_1.exclusions(),
artifactId: "beam-vendor-grpc-1_48_1",
groupId: group,
version: version,
)

0 comments on commit db8db0b

Please sign in to comment.