forked from apache/beam
-
Notifications
You must be signed in to change notification settings - Fork 0
/
build.gradle
175 lines (162 loc) · 7.42 KB
/
build.gradle
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import groovy.json.JsonOutput
plugins {
id 'java'
id 'org.apache.beam.module'
id 'com.github.johnrengelman.shadow'
}
applyJavaNature(
exportJavadoc: false,
automaticModuleName: 'org.apache.beam.examples',
)
provideIntegrationTestingDependencies()
enableJavaPerformanceTesting()
description = "Apache Beam :: Examples :: Java"
ext.summary = """Apache Beam SDK provides a simple, Java-based
interface for processing virtually any size data. This
artifact includes all Apache Beam Java SDK examples."""
/** Define the list of runners which execute a precommit test.
* Some runners are run from separate projects, see the preCommit task below
* for details.
*/
def preCommitRunners = ["directRunner", "flinkRunner", "sparkRunner"]
for (String runner : preCommitRunners) {
configurations.create(runner + "PreCommit")
}
configurations.sparkRunnerPreCommit {
// Ban certain dependencies to prevent a StackOverflow within Spark
// because JUL -> SLF4J -> JUL, and similarly JDK14 -> SLF4J -> JDK14
exclude group: "org.slf4j", module: "jul-to-slf4j"
exclude group: "org.slf4j", module: "slf4j-jdk14"
}
dependencies {
implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
implementation library.java.vendored_guava_32_1_2_jre
implementation library.java.kafka_clients
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation project(":sdks:java:extensions:avro")
implementation project(":sdks:java:extensions:google-cloud-platform-core")
implementation project(":sdks:java:extensions:python")
implementation project(":sdks:java:io:google-cloud-platform")
implementation project(":sdks:java:io:kafka")
implementation project(":sdks:java:extensions:ml")
implementation library.java.avro
implementation library.java.bigdataoss_util
implementation library.java.google_api_client
implementation library.java.google_api_services_bigquery
implementation library.java.google_api_services_pubsub
implementation library.java.google_auth_library_credentials
implementation library.java.google_auth_library_oauth2_http
implementation library.java.google_cloud_datastore_v1_proto_client
implementation library.java.google_code_gson
implementation library.java.google_http_client
implementation library.java.google_oauth_client
implementation library.java.jackson_databind
implementation library.java.joda_time
implementation library.java.protobuf_java
implementation library.java.proto_google_cloud_bigtable_v2
implementation library.java.proto_google_cloud_datastore_v1
implementation library.java.slf4j_api
implementation library.java.commons_io
implementation library.java.commons_csv
runtimeOnly project(path: ":runners:direct-java", configuration: "shadow")
implementation library.java.vendored_grpc_1_54_0
implementation library.java.vendored_guava_32_1_2_jre
implementation "com.google.api.grpc:proto-google-cloud-language-v1:1.81.4"
implementation ("io.confluent:kafka-avro-serializer:5.3.2") {
// It depends on "spotbugs-annotations:3.1.9" which clashes with current
// "spotbugs-annotations:3.1.12" used in Beam. Not required.
exclude group: "org.apache.zookeeper", module: "zookeeper"
}
implementation "org.apache.commons:commons-lang3:3.9"
implementation "org.apache.httpcomponents:httpclient:4.5.13"
implementation "org.apache.httpcomponents:httpcore:4.4.13"
implementation "com.fasterxml.jackson.core:jackson-annotations:2.14.1"
implementation "com.fasterxml.jackson.core:jackson-core:2.14.1"
testImplementation project(path: ":runners:direct-java", configuration: "shadow")
testImplementation project(":sdks:java:io:google-cloud-platform")
testImplementation project(":sdks:java:extensions:ml")
testImplementation library.java.google_cloud_bigquery
testImplementation library.java.hamcrest
testImplementation library.java.junit
testImplementation library.java.mockito_core
testImplementation library.java.testcontainers_kafka
testImplementation library.java.testcontainers_gcloud
// Add dependencies for the PreCommit configurations
// For each runner a project level dependency on the examples project.
for (String runner : preCommitRunners) {
delegate.add(runner + "PreCommit", project(path: ":examples:java", configuration: "testRuntimeMigration"))
}
directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow")
flinkRunnerPreCommit project(":runners:flink:${project.ext.latestFlinkVersion}")
sparkRunnerPreCommit project(":runners:spark:3")
sparkRunnerPreCommit project(":sdks:java:io:hadoop-file-system")
// Add dependency if requested on command line for runner
if (project.hasProperty("runnerDependency")) {
runtimeOnly project(path: project.getProperty("runnerDependency"))
}
}
/*
* Create a ${runner}PreCommit task for each runner which runs a set
* of integration tests for WordCount and WindowedWordCount.
*/
def preCommitRunnerClass = [
directRunner: "org.apache.beam.runners.direct.DirectRunner",
flinkRunner: "org.apache.beam.runners.flink.TestFlinkRunner",
sparkRunner: "org.apache.beam.runners.spark.TestSparkRunner",
]
def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
def gcsTempRoot = project.findProperty('gcsTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests/'
for (String runner : preCommitRunners) {
tasks.create(name: runner + "PreCommit", type: Test) {
def preCommitBeamTestPipelineOptions = [
"--project=${gcpProject}",
"--tempRoot=${gcsTempRoot}",
"--runner=" + preCommitRunnerClass[runner],
]
classpath = configurations."${runner}PreCommit"
include "**/WordCountIT.class"
if (!"sparkRunner".equals(runner)) {
include "**/WindowedWordCountIT.class"
}
forkEvery 1
maxParallelForks 4
systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions)
}
}
/* Define a common precommit task which depends on all the individual precommits. */
task preCommit() {
for (String runner : preCommitRunners) {
dependsOn runner + "PreCommit"
}
}
tasks.create(name:"execute", type:JavaExec) {
main = project.hasProperty("mainClass") ? project.getProperty("mainClass") : "NONE"
classpath = sourceSets.main.runtimeClasspath
systemProperties System.getProperties()
args project.hasProperty("exec.args") ? project.getProperty("exec.args").split() : []
}
// Run this task to validate the Java environment setup for contributors
task wordCount(type:JavaExec) {
description "Run the Java word count example"
mainClass = "org.apache.beam.examples.WordCount"
classpath = sourceSets.main.runtimeClasspath
systemProperties = System.getProperties()
args = ["--output=/tmp/ouput.txt"]
}