forked from apache/beam
-
Notifications
You must be signed in to change notification settings - Fork 0
/
build.gradle
262 lines (232 loc) · 13 KB
/
build.gradle
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
plugins { id 'org.apache.beam.module' }
// Set a specific version of 'com.google.apis:google-api-services-dataflow'
// by adding -Pdataflow.version=<version> in Gradle command. Otherwise,
// 'google_clients_version' defined in BeamModulePlugin will be used as default.
def DATAFLOW_VERSION = "dataflow.version"
// Get full dependency of 'com.google.apis:google-api-services-dataflow'
def google_api_services_dataflow = project.hasProperty(DATAFLOW_VERSION) ? "com.google.apis:google-api-services-dataflow:" + getProperty(DATAFLOW_VERSION) : library.java.google_api_services_dataflow
// Returns a string representing the relocated path to be used with the shadow
// plugin when given a suffix such as "com.".
def getWorkerRelocatedPath = { String suffix ->
return ("org.apache.beam.runners.dataflow.worker.repackaged."
+ suffix)
}
def sdk_provided_dependencies = [
google_api_services_dataflow,
library.java.avro,
library.java.google_http_client,
library.java.google_http_client_gson,
library.java.jackson_annotations,
library.java.jackson_core,
library.java.jackson_databind,
library.java.joda_time,
library.java.slf4j_api,
library.java.vendored_grpc_1_54_0,
]
def sdk_provided_shaded_project_dependencies = [
":model:pipeline",
":sdks:java:core",
]
def sdk_provided_project_dependencies = [
":runners:google-cloud-dataflow-java",
":sdks:java:extensions:avro",
":sdks:java:extensions:google-cloud-platform-core",
":sdks:java:io:google-cloud-platform",
]
// Exclude unneeded dependencies when building jar
def excluded_dependencies = [
"com.google.auto.service:auto-service", // Provided scope added from applyJavaNature
"com.google.auto.value:auto-value", // Provided scope added from applyJavaNature
"org.codehaus.jackson:jackson-core-asl", // Exclude an old version of jackson-core-asl introduced by google-http-client-jackson
"org.objenesis:objenesis", // Transitive dependency introduced from Beam
library.java.commons_compress, // Transitive dependency introduced from Beam
library.java.error_prone_annotations, // Provided scope added in worker
library.java.hamcrest, // Test only
library.java.junit, // Test only
library.java.jsonassert, // Test only
library.java.truth // Test only
]
applyJavaNature(
automaticModuleName: 'org.apache.beam.runners.dataflow.worker',
archivesBaseName: 'beam-runners-google-cloud-dataflow-java-legacy-worker',
classesTriggerCheckerBugs: [
'BatchGroupAlsoByWindowAndCombineFn' : 'TODO: file a bug report',
'AssignWindowsParDoFnFactory' : 'TODO: file a bug report',
'FetchAndFilterStreamingSideInputsOperation': 'https://github.com/typetools/checker-framework/issues/5436',
],
exportJavadoc: false,
enableSpotbugs: false /* TODO(BEAM-5658): enable spotbugs */,
shadowJarValidationExcludes: [
"org/apache/beam/runners/dataflow/worker/**",
"org/apache/beam/repackaged/beam_runners_google_cloud_dataflow_java_legacy_worker/**",
// TODO(https://github.com/apache/beam/issues/19114): Move DataflowRunnerHarness class under org.apache.beam.runners.dataflow.worker namespace
"com/google/cloud/dataflow/worker/DataflowRunnerHarness.class",
// Allow slf4j implementation worker for logging during pipeline execution
"org/slf4j/impl/**"
],
shadowClosure: {
// Each included dependency must also include all of its necessary transitive dependencies
// or have them provided by the users pipeline during job submission. Typically a users
// pipeline includes :runners:google-cloud-dataflow-java and its transitive dependencies
// so those dependencies don't need to be shaded (bundled and relocated) away. All other
// dependencies needed to run the worker must be shaded (bundled and relocated) to prevent
// ClassNotFound and/or MethodNotFound errors during pipeline execution.
//
// Each included dependency should have a matching relocation rule below that ensures
// that the shaded jar is correctly built.
dependencies {
include(dependency(library.java.slf4j_jdk14))
}
dependencies {
include(project(path: ":model:fn-execution", configuration: "shadow"))
}
relocate("org.apache.beam.model.fnexecution.v1", getWorkerRelocatedPath("org.apache.beam.model.fnexecution.v1"))
dependencies {
include(project(":runners:core-construction-java"))
include(project(":runners:core-java"))
}
relocate("org.apache.beam.runners.core", getWorkerRelocatedPath("org.apache.beam.runners.core"))
relocate("org.apache.beam.repackaged.beam_runners_core_construction_java", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_core_construction_java"))
relocate("org.apache.beam.repackaged.beam_runners_core_java", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_core_java"))
dependencies {
include(project(":runners:java-fn-execution"))
}
relocate("org.apache.beam.runners.fnexecution", getWorkerRelocatedPath("org.apache.beam.runners.fnexecution"))
relocate("org.apache.beam.repackaged.beam_runners_java_fn_execution", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_java_fn_execution"))
dependencies {
include(project(":sdks:java:fn-execution"))
}
relocate("org.apache.beam.sdk.fn", getWorkerRelocatedPath("org.apache.beam.sdk.fn"))
relocate("org.apache.beam.repackaged.beam_sdks_java_fn_execution", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_sdks_java_fn_execution"))
dependencies {
// We have to include jetty-server/jetty-servlet and all of its transitive dependencies
// which includes several org.eclipse.jetty artifacts + servlet-api
include(dependency("org.eclipse.jetty:.*:9.2.10.v20150310"))
include(dependency("javax.servlet:javax.servlet-api:3.1.0"))
}
relocate("org.eclipse.jetty", getWorkerRelocatedPath("org.eclipse.jetty"))
relocate("javax.servlet", getWorkerRelocatedPath("javax.servlet"))
// We don't relocate windmill since it is already underneath the org.apache.beam.runners.dataflow.worker namespace and never
// expect a user pipeline to include it. There is also a JNI component that windmill server relies on which makes
// arbitrary relocation more difficult.
dependencies {
include(project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow"))
}
// Include original source files extracted under
// '$buildDir/original_sources_to_package' to jar
from "$buildDir/original_sources_to_package"
exclude "META-INF/LICENSE.txt"
exclude "about.html"
})
/******************************************************************************/
// Configure the worker root project
configurations {
sourceFile
// Ban these dependencies from all configurations
all {
// Ban the usage of AppleJavaExtensions in spotbugs.
exclude group: "com.apple", module: "AppleJavaExtensions"
}
}
dependencies {
implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
// Note that any dependency that is modified here should also be modified within
// runners/google-cloud-dataflow-java/worker/build.gradle using the rules provided
// there.
sdk_provided_dependencies.each {
provided(it)
testImplementation(it)
}
sdk_provided_shaded_project_dependencies.each {
provided project(path: it, configuration: "shadow")
}
sdk_provided_project_dependencies.each {
provided project(path: it)
testImplementation project(path: it)
}
implementation project(path: ":model:fn-execution", configuration: "shadow")
implementation project(path: ":model:pipeline", configuration: "shadow")
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation project(":runners:core-construction-java")
implementation project(":runners:core-java")
implementation project(":runners:java-fn-execution")
implementation project(":sdks:java:fn-execution")
implementation project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow")
shadow library.java.vendored_guava_32_1_2_jre
implementation library.java.google_auth_library_credentials
implementation library.java.proto_google_common_protos
// Conscrypt shouldn't be included here because Conscrypt won't work when being shaded.
// (Context: https://github.com/apache/beam/pull/13846)
// Conscrypt will be added to runtime dependencies by GrpcVendoring so compileOnly works for now.
compileOnly "org.conscrypt:conscrypt-openjdk-uber:2.5.1"
implementation "javax.servlet:javax.servlet-api:3.1.0"
implementation "org.eclipse.jetty:jetty-server:9.2.10.v20150310"
implementation "org.eclipse.jetty:jetty-servlet:9.2.10.v20150310"
implementation library.java.avro
implementation library.java.jackson_annotations
implementation library.java.jackson_core
implementation library.java.jackson_databind
implementation library.java.joda_time
implementation library.java.slf4j_api
implementation library.java.vendored_grpc_1_54_0
implementation library.java.error_prone_annotations
permitUnusedDeclared library.java.error_prone_annotations
runtimeOnly library.java.slf4j_jdk14
// Any test dependency which intersects with our relocation rules above needs to be relocated
// as well and placed within the testImplementation configuration. Otherwise we can place it within
// the shadowTest configuration.
testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration")
// TODO: excluding Guava until Truth updates it to >32.1.x
testImplementation(library.java.truth) {
exclude group: 'com.google.guava', module: 'guava'
}
shadowTest project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntimeMigration")
shadowTest project(path: ":runners:direct-java", configuration: "shadow")
shadowTest project(path: ":sdks:java:harness", configuration: "shadowTest")
shadowTest project(path: ":sdks:java:core", configuration: "shadowTest")
shadowTest library.java.jsonassert
shadowTest library.java.junit
shadowTest library.java.mockito_core
shadowTest library.java.powermock
}
project.task('validateShadedJarContainsSlf4jJdk14', dependsOn: 'shadowJar') {
ext.outFile = project.file("${project.reportsDir}/${name}.out")
inputs.files(project.configurations.shadow.artifacts.files)
.withPropertyName("shadowArtifactsFiles")
.withPathSensitivity(PathSensitivity.RELATIVE)
outputs.files outFile
doLast {
project.configurations.shadow.artifacts.files.each {
FileTree slf4jImpl = project.zipTree(it).matching {
include "org/slf4j/impl/JDK14LoggerAdapter.class"
}
outFile.text = slf4jImpl.files
if (slf4jImpl.files.isEmpty()) {
throw new GradleException("Did not find slf4j-jdk14 in Dataflow Worker uber jar")
}
}
}
}
tasks.check.dependsOn project.tasks.validateShadedJarContainsSlf4jJdk14
//TODO(https://github.com/apache/beam/issues/19115): checktyle task should be enabled in the future.
checkstyleMain.enabled = false
checkstyleTest.enabled = false
//TODO(https://github.com/apache/beam/issues/19119): javadoc task should be enabled in the future.
javadoc.enabled = false