Skip to content

Commit

Permalink
[BEAM-4046] decouple gradle project names and maven artifact ids
Browse files Browse the repository at this point in the history
  • Loading branch information
adude3141 committed May 11, 2019
1 parent c02af60 commit 1e7ea0d
Show file tree
Hide file tree
Showing 159 changed files with 931 additions and 1,011 deletions.
8 changes: 4 additions & 4 deletions .test-infra/jenkins/CommonTestProperties.groovy
Expand Up @@ -32,10 +32,10 @@ class CommonTestProperties {

def RUNNER_DEPENDENCY_MAP = [
JAVA: [
DATAFLOW: ":beam-runners-google-cloud-dataflow-java",
SPARK: ":beam-runners-spark",
FLINK: ":beam-runners-flink_2.11",
DIRECT: ":beam-runners-direct-java"
DATAFLOW: ":runners:google-cloud-dataflow-java",
SPARK: ":runners:spark",
FLINK: ":runners:flink:1.5",
DIRECT: ":runners:direct-java"
],
PYTHON: [
DATAFLOW: "TestDataflowRunner",
Expand Down
4 changes: 2 additions & 2 deletions .test-infra/jenkins/LoadTestsBuilder.groovy
Expand Up @@ -48,9 +48,9 @@ class LoadTestsBuilder {

private static String getGradleTaskName(SDK sdk) {
if (sdk == SDK.JAVA) {
return ':beam-sdks-java-load-tests:run'
return ':beam:sdks:java:load-tests:run'
} else if (sdk == SDK.PYTHON) {
return ':beam-sdks-python-load-tests:run'
return ':sdks:python:apache_beam:testing:load-tests:run'
} else {
throw new RuntimeException("No task name defined for SDK: $SDK")
}
Expand Down
2 changes: 1 addition & 1 deletion .test-infra/jenkins/NexmarkBuilder.groovy
Expand Up @@ -79,7 +79,7 @@ class NexmarkBuilder {
shell("echo *** RUN ${title} ***")
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches("-Pnexmark.runner=${runner.getDepenedencyBySDK(sdk)}")
switches("-Pnexmark.args=\"${parseOptions(options)}\"")
Expand Down
Expand Up @@ -34,7 +34,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_ValidatesRunner_Porta
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-google-cloud-dataflow-java:validatesRunnerFnApiWorkerTest')
tasks(':runners:google-cloud-dataflow-java:validatesRunnerFnApiWorkerTest')
switches ('-Pdockerfile=Dockerfile-java11')

// Increase parallel worker threads above processor limit since most time is
Expand Down
16 changes: 8 additions & 8 deletions .test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow.groovy
Expand Up @@ -38,9 +38,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN BATCH MODE USING DATAFLOW RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-google-cloud-dataflow-java"' +
switches('-Pnexmark.runner=":runners:google-cloud-dataflow-java"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--runner=DataflowRunner',
Expand All @@ -58,9 +58,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN STREAMING MODE USING DATAFLOW RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-google-cloud-dataflow-java"' +
switches('-Pnexmark.runner=":runners:google-cloud-dataflow-java"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--runner=DataflowRunner',
Expand All @@ -78,9 +78,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN SQL BATCH MODE USING DATAFLOW RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-google-cloud-dataflow-java"' +
switches('-Pnexmark.runner=":runners:google-cloud-dataflow-java"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--runner=DataflowRunner',
Expand All @@ -99,9 +99,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN SQL STREAMING MODE USING DATAFLOW RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-google-cloud-dataflow-java"' +
switches('-Pnexmark.runner=":runners:google-cloud-dataflow-java"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--runner=DataflowRunner',
Expand Down
16 changes: 8 additions & 8 deletions .test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
Expand Up @@ -37,9 +37,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN BATCH MODE USING DIRECT RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-direct-java"' +
switches('-Pnexmark.runner=":runners:direct-java"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--runner=DirectRunner',
Expand All @@ -53,9 +53,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN STREAMING MODE USING DIRECT RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-direct-java"' +
switches('-Pnexmark.runner=":runners:direct-java"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--runner=DirectRunner',
Expand All @@ -69,9 +69,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN SQL BATCH MODE USING DIRECT RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-direct-java"' +
switches('-Pnexmark.runner=":runners:direct-java"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--runner=DirectRunner',
Expand All @@ -86,9 +86,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN SQL STREAMING MODE USING DIRECT RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-direct-java"' +
switches('-Pnexmark.runner=":runners:direct-java"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--runner=DirectRunner',
Expand Down
16 changes: 8 additions & 8 deletions .test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
Expand Up @@ -38,9 +38,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN BATCH MODE USING FLINK RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-flink_2.11"' +
switches('-Pnexmark.runner=":runners:flink:1.5"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--streaming=false',
Expand All @@ -53,9 +53,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN STREAMING MODE USING FLINK RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-flink_2.11"' +
switches('-Pnexmark.runner=":runners:flink:1.5"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--streaming=true',
Expand All @@ -68,9 +68,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN SQL BATCH MODE USING FLINK RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-flink_2.11"' +
switches('-Pnexmark.runner=":runners:flink:1.5"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--queryLanguage=sql',
Expand All @@ -83,9 +83,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN SQL STREAMING MODE USING FLINK RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-flink_2.11"' +
switches('-Pnexmark.runner=":runners:flink:1.5"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--queryLanguage=sql',
Expand Down
8 changes: 4 additions & 4 deletions .test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
Expand Up @@ -36,9 +36,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK IN BATCH MODE USING SPARK RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-spark"' +
switches('-Pnexmark.runner=":runners:spark"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--runner=SparkRunner',
Expand All @@ -51,9 +51,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
shell('echo *** RUN NEXMARK SQL IN BATCH MODE USING SPARK RUNNER ***')
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-java-nexmark:run')
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
switches('-Pnexmark.runner=":beam-runners-spark"' +
switches('-Pnexmark.runner=":runners:spark"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--runner=SparkRunner',
Expand Down
Expand Up @@ -36,7 +36,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Flink_Batch',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-flink_2.11-job-server:validatesPortableRunnerBatch')
tasks(':runners:flink:1.5:job-server:validatesPortableRunnerBatch')
commonJobProperties.setGradleSwitches(delegate)
}
}
Expand Down
Expand Up @@ -36,7 +36,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Flink_Streaming',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-flink_2.11-job-server:validatesPortableRunnerStreaming')
tasks(':runners:flink:1.5:job-server:validatesPortableRunnerStreaming')
commonJobProperties.setGradleSwitches(delegate)
}
}
Expand Down
Expand Up @@ -37,7 +37,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Apex',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-apex:validatesRunner')
tasks(':runners:apex:validatesRunner')
commonJobProperties.setGradleSwitches(delegate)
}
}
Expand Down
Expand Up @@ -40,7 +40,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflo
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-google-cloud-dataflow-java:validatesRunner')
tasks(':runners:google-cloud-dataflow-java:validatesRunner')
// Increase parallel worker threads above processor limit since most time is
// spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
// because each one launches a Dataflow job with about 3 mins of overhead.
Expand Down
Expand Up @@ -39,7 +39,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflo
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-google-cloud-dataflow-java:validatesRunnerFnApiWorkerExecutableStageTest')
tasks(':runners:google-cloud-dataflow-java:validatesRunnerFnApiWorkerExecutableStageTest')
// Increase parallel worker threads above processor limit since most time is
// spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
// because each one launches a Dataflow job with about 3 mins of overhead.
Expand Down
Expand Up @@ -34,7 +34,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_ValidatesRunner_Dataf
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-google-cloud-dataflow-java:validatesJava11Runner')
tasks(':runners:google-cloud-dataflow-java:validatesJava11Runner')
// Increase parallel worker threads above processor limit since most time is
// spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
// because each one launches a Dataflow job with about 3 mins of overhead.
Expand Down
Expand Up @@ -38,14 +38,14 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_ValidatesRunner_Direc
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-direct-java:shadowJar')
tasks(':beam-runners-direct-java:shadowTestJar')
tasks(':runners:direct-java:shadowJar')
tasks(':runners:direct-java:shadowTestJar')
switches("-Dorg.gradle.java.home=${JAVA_8_HOME}")
}

gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-direct-java:validatesRunner')
tasks(':runners:direct-java:validatesRunner')
switches("-Dorg.gradle.java.home=${JAVA_11_HOME}")
switches('-x shadowJar')
switches('-x shadowTestJar')
Expand Down
Expand Up @@ -37,7 +37,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Flink',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-flink_2.11:validatesRunner')
tasks(':runners:flink:1.5:validatesRunner')
commonJobProperties.setGradleSwitches(delegate)
}
}
Expand Down
Expand Up @@ -41,7 +41,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Gearpum
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-gearpump:validatesRunner')
tasks(':runners:gearpump:validatesRunner')
commonJobProperties.setGradleSwitches(delegate)
}
}
Expand Down
Expand Up @@ -40,7 +40,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Portabi
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-google-cloud-dataflow-java:validatesRunnerFnApiWorkerTest')
tasks(':runners:google-cloud-dataflow-java:validatesRunnerFnApiWorkerTest')
// Increase parallel worker threads above processor limit since most time is
// spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
// because each one launches a Dataflow job with about 3 mins of overhead.
Expand Down
Expand Up @@ -37,7 +37,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Samza',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-samza:validatesRunner')
tasks(':unners:samza:validatesRunner')
commonJobProperties.setGradleSwitches(delegate)
}
}
Expand Down
Expand Up @@ -37,7 +37,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Spark',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-runners-spark:validatesRunner')
tasks(':runners:spark:validatesRunner')
commonJobProperties.setGradleSwitches(delegate)
}
}
Expand Down
Expand Up @@ -32,8 +32,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow', 'Run Python
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-python:validatesRunnerBatchTests')
tasks(':beam-sdks-python:validatesRunnerStreamingTests')
tasks(':sdks:python:validatesRunnerBatchTests')
tasks(':sdks:python:validatesRunnerStreamingTests')
commonJobProperties.setGradleSwitches(delegate)
}
}
Expand Down
4 changes: 2 additions & 2 deletions .test-infra/jenkins/job_PostCommit_Website_Publish.groovy
Expand Up @@ -33,8 +33,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Website_Publish', '',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-website:clean')
tasks(':beam-website:publishWebsite')
tasks(':website:clean')
tasks(':website:publishWebsite')
commonJobProperties.setGradleSwitches(delegate)
}
}
Expand Down
Expand Up @@ -23,7 +23,7 @@ import PrecommitJobBuilder
PrecommitJobBuilder builder = new PrecommitJobBuilder(
scope: this,
nameBase: 'Python_PVR_Flink',
gradleTask: ':beam-sdks-python:flinkValidatesRunner',
gradleTask: ':sdks:python:flinkValidatesRunner',
triggerPathPatterns: [
'^model/.*$',
'^runners/.*$',
Expand Down
2 changes: 1 addition & 1 deletion .test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy
Expand Up @@ -21,7 +21,7 @@ import PrecommitJobBuilder
PrecommitJobBuilder builder = new PrecommitJobBuilder(
scope: this,
nameBase: 'Website_Stage_GCS',
gradleTask: ':beam-website:stageWebsite',
gradleTask: ':website:stageWebsite',
triggerPathPatterns: ['^website/.*$']
)
builder.build {
Expand Down
4 changes: 2 additions & 2 deletions .test-infra/jenkins/job_Release_Python_NightlySnapshot.groovy
Expand Up @@ -44,13 +44,13 @@ job('beam_Release_Python_NightlySnapshot') {
// Cleanup Python directory.
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-python:clean')
tasks(':sdks:python:clean')
commonJobProperties.setGradleSwitches(delegate)
}
// Build snapshot.
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':beam-sdks-python:buildSnapshot')
tasks(':sdks:python:buildSnapshot')
commonJobProperties.setGradleSwitches(delegate)
}
// Publish snapshot to a public accessible GCS directory.
Expand Down

0 comments on commit 1e7ea0d

Please sign in to comment.