[BEAM-5634] Bring dataflow java worker code into beam

runners/google-cloud-dataflow-java/worker/build.gradle

@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/******************************************************************************/
+// Apply BeamModulePlugin
+
+// Reuse project_root/buildSrc in this build.gradle file to reduce the
+// maintenance burden and simpily this file. See BeamModulePlugin for
+// documentation on default build tasks and properties that are enabled in
+// addition to natures that will be applied to worker.
+apply plugin: org.apache.beam.gradle.BeamModulePlugin
+
+group = "org.apache.beam.runners.dataflow"
+
+/******************************************************************************/
+// Apply Java nature with customized configurations
+
+// Set a specific version of 'com.google.apis:google-api-services-dataflow'
+// by adding -Pdataflow.version=<version> in Gradle command. Otherwise,
+// 'google_clients_version' defined in BeamModulePlugin will be used as default.
+def DATAFLOW_VERSION = "dataflow.version"
+
+// To build FnAPI or legacy worker.
+// Use -PisLegacyWorker in Gradle command if build legacy worker, otherwise,
+// FnAPI worker is considered as default.
+def is_legacy_worker = {
+  return project.hasProperty("isLegacyWorker")
+}
+
+// Get full dependency of 'com.google.apis:google-api-services-dataflow'
+def google_api_services_dataflow = project.hasProperty(DATAFLOW_VERSION) ? "com.google.apis:google-api-services-dataflow:" + getProperty(DATAFLOW_VERSION) : library.java.google_api_services_dataflow
+
+// Returns a string representing the relocated path to be used with the shadow
+// plugin when given a suffix such as "com.".
+def getWorkerRelocatedPath = { String suffix ->
+  return ("org.apache.beam.runners.dataflow.worker.repackaged."
+          + suffix)
+}
+
+// Following listed dependencies will be shaded only in fnapi worker, not legacy
+// worker
+def sdk_provided_dependencies = [
+  "org.apache.beam:beam-runners-google-cloud-dataflow-java:$version",
+  "org.apache.beam:beam-sdks-java-core:$version",
+  "org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:$version",
+  "org.apache.beam:beam-sdks-java-io-google-cloud-platform:$version",
+  google_api_services_dataflow,
+  library.java.avro,
+  library.java.google_api_client,
+  library.java.google_http_client,
+  library.java.google_http_client_jackson,
+  library.java.jackson_annotations,
+  library.java.jackson_core,
+  library.java.jackson_databind,
+  library.java.joda_time,
+]
+
+// Exclude unneeded dependencies when building jar
+def excluded_dependencies = [
+  "com.google.auto.service:auto-service",      // Provided scope added from applyJavaNature
+  "com.google.auto.value:auto-value",          // Provided scope added from applyJavaNature
+  "org.codehaus.jackson:jackson-core-asl",     // Exclude an old version of jackson-core-asl introduced by google-http-client-jackson
+  "org.objenesis:objenesis",                   // Transitive dependency introduced from Beam
+  "org.tukaani:xz",                            // Transitive dependency introduced from Beam
+  library.java.commons_compress,               // Transitive dependency introduced from Beam
+  library.java.error_prone_annotations,        // Provided scope added in worker
+  library.java.hamcrest_core,                  // Test only
+  library.java.hamcrest_library,               // Test only
+  library.java.junit,                          // Test only
+]
+
+applyJavaNature(validateShadowJar: false, shadowClosure: DEFAULT_SHADOW_CLOSURE << {
+  dependencies {
+    include(project(path: ":beam-runners-google-cloud-dataflow-java-windmill", configuration: "shadow"))
+    include(dependency(".*:.*"))
+
+    if (is_legacy_worker()) {
+      sdk_provided_dependencies.each {
+        exclude(dependency(it))
+      }
+      excluded_dependencies.each {
+        exclude(dependency(it))
+      }
+    }
+  }
+
+  // Archive name pattern: ${name}-${appendix}-${version}-${classifier}.jar
+  appendix = is_legacy_worker() ? "legacy-bundled" : "fnapi-bundled"
+
+  // Include original source files extracted under
+  // '$buildDir/original_sources_to_package' to jar
+  from "$buildDir/original_sources_to_package"
+
+  exclude "META-INF/LICENSE.txt"
+  exclude "about.html"
+  exclude "google/protobuf/*.proto"
+  exclude "windmill*.proto"
+
+  if (is_legacy_worker()) {
+    relocate("com.", getWorkerRelocatedPath("com.")) {
+      exclude "com.fasterxml.jackson.**"
+      exclude "com.google.api.client.**"
+      exclude "com.google.api.services.bigquery.**"
+      exclude "com.google.api.services.clouddebugger.**"
+      exclude "com.google.api.services.dataflow.**"
+      exclude "com.google.api.services.datastore.**"
+      exclude "com.google.api.services.pubsub.**"
+      exclude "com.google.api.services.storage.**"
+      exclude "com.google.auth.**"
+      exclude "com.google.cloud.dataflow.**"
+      exclude "com.sun.management*"
+      exclude "com.sun.management.**"
+    }
+    relocate("javax.servlet", getWorkerRelocatedPath("javax.servlet"))
+    relocate("io.", getWorkerRelocatedPath("io."))
+    relocate("okio.", getWorkerRelocatedPath("okio."))
+    relocate("org.", getWorkerRelocatedPath("org.")) {
+      // Exclude netty-tcnative from shading since gRPC relies on Netty to be able
+      // to load org.apache.tomcat.jni.SSL to provide an SSL context.
+      exclude "org.apache.avro.**"
+      exclude "org.apache.beam.**"
+      exclude "org.apache.tomcat.jni.**"
+      exclude "org.conscrypt.**"
+      exclude "org.eclipse.jetty.alpn.**"
+      exclude "org.eclipse.jetty.npn.**"
+      exclude "org.hamcrest.**"
+      exclude "org.joda.time.**"
+      exclude "org.junit.**"
+      exclude "org.slf4j.**"
+      exclude "org.w3c.dom.**"
+    }
+    relocate("org.apache.beam.runners.core.construction.",
+             getWorkerRelocatedPath("org."))
+  }
+})
+
+/******************************************************************************/
+// Configure the worker root project
+
+configurations {
+  sourceFile
+
+  // Ban these dependencies from all configurations
+  all {
+    // Ban the usage of AppleJavaExtensions in findbugs.
+    exclude group: "com.apple", module: "AppleJavaExtensions"
+  }
+}
+
+dependencies {
+  // Set dependencies to shadow scope by default, but in a property so they can
+  // be downgraded when building a legacy (non-FnAPI) worker.
+  if (is_legacy_worker()) {
+    sdk_provided_dependencies.each {
+      provided(it)
+    }
+  } else {
+    sdk_provided_dependencies.each {
+      shadow(it)
+    }
+  }
+
+  compile "org.apache.beam:beam-model-fn-execution:$version"
+  compile "org.apache.beam:beam-model-pipeline:$version"
+  compile "org.apache.beam:beam-runners-core-construction-java:$version"
+  compile "org.apache.beam:beam-runners-core-java:$version"
+  compile "org.apache.beam:beam-runners-java-fn-execution:$version"
+  compile "org.apache.beam:beam-sdks-java-fn-execution:$version"
+  compile project(path: ":beam-runners-google-cloud-dataflow-java-windmill", configuration: "shadow")
+
+  compile library.java.guava
+  compile library.java.slf4j_api
+  compile "javax.servlet:javax.servlet-api:3.1.0"
+  compile "org.conscrypt:conscrypt-openjdk:1.1.3:linux-x86_64"
+  compile "org.eclipse.jetty:jetty-server:9.2.10.v20150310"
+  compile "org.eclipse.jetty:jetty-servlet:9.2.10.v20150310"
+
+  provided library.java.error_prone_annotations
+
+  runtime library.java.slf4j_jdk14
+
+  testCompile "org.apache.beam:beam-runners-core-java:$version:tests"
+  testCompile "org.apache.beam:beam-runners-direct-java:$version"
+  testCompile "org.apache.beam:beam-sdks-java-core:$version:tests"
+  testCompile "org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:$version:tests"
+
+  testCompile library.java.guava_testlib
+  testCompile library.java.hamcrest_core
+  testCompile library.java.hamcrest_library
+  testCompile library.java.junit
+  testCompile library.java.mockito_core
+}
+
+//TODO(BEAM-5657): checktyle task should be enabled in the future.
+checkstyleMain.enabled = false
+checkstyleTest.enabled = false
+//TODO(BEAM-5658): fingbugs task should be enabled in the future.
+findbugsMain.enabled = false
+//TODO(BEAM-5659): javadoc task should be enabled in the future.
+javadoc.enabled = false

runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/harness/util/ThrowingBiConsumer.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.dataflow.harness.util;
+
+import java.util.function.BiConsumer;
+
+/**
+ * A {@link BiConsumer} which can throw {@link Exception}s.
+ *
+ * <p>Used to expand the allowed set of method references to be used by Java 8 functional
+ * interfaces.
+ */
+@FunctionalInterface
+public interface ThrowingBiConsumer<T1, T2> {
+  void accept(T1 t1, T2 t2) throws Exception;
+}

runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/harness/util/ThrowingBiFunction.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.dataflow.harness.util;
+
+import java.util.function.BiFunction;
+
+/**
+ * A {@link BiFunction} which can throw {@link Exception}s.
+ *
+ * <p>Used to expand the allowed set of method references to be used by Java 8 functional
+ * interfaces.
+ */
+@FunctionalInterface
+public interface ThrowingBiFunction<T1, T2, T3> {
+  T3 apply(T1 t1, T2 t2) throws Exception;
+}

runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/harness/util/ThrowingConsumer.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.dataflow.harness.util;
+
+import java.util.function.Consumer;
+
+/**
+ * A {@link Consumer} which can throw {@link Exception}s.
+ *
+ * <p>Used to expand the allowed set of method references to be used by Java 8 functional
+ * interfaces.
+ */
+@FunctionalInterface
+public interface ThrowingConsumer<T> {
+  void accept(T t) throws Exception;
+}

runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/harness/util/ThrowingFunction.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.dataflow.harness.util;
+
+import java.util.function.Function;
+
+/**
+ * A {@link Function} which can throw {@link Exception}s.
+ *
+ * <p>Used to expand the allowed set of method references to be used by Java 8 functional
+ * interfaces.
+ */
+@FunctionalInterface
+public interface ThrowingFunction<T1, T2> {
+  T2 apply(T1 value) throws Exception;
+}

runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/harness/util/ThrowingRunnable.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.dataflow.harness.util;
+
+/**
+ * A {@link Runnable} which can throw {@link Exception}s.
+ *
+ * <p>Used to expand the allowed set of method references to be used by Java 8 functional
+ * interfaces.
+ */
+@FunctionalInterface
+public interface ThrowingRunnable {
+  void run() throws Exception;
+}