From 1e7ea0da5073566c3fa26dbc1105105fbe6043ae Mon Sep 17 00:00:00 2001 From: Michael Luckey <25622840+adude3141@users.noreply.github.com> Date: Tue, 2 Apr 2019 03:19:32 +0200 Subject: [PATCH] [BEAM-4046] decouple gradle project names and maven artifact ids --- .../jenkins/CommonTestProperties.groovy | 8 +- .test-infra/jenkins/LoadTestsBuilder.groovy | 4 +- .test-infra/jenkins/NexmarkBuilder.groovy | 2 +- ...datesRunner_PortabilityApi_Dataflow.groovy | 2 +- ...ob_PostCommit_Java_Nexmark_Dataflow.groovy | 16 +- .../job_PostCommit_Java_Nexmark_Direct.groovy | 16 +- .../job_PostCommit_Java_Nexmark_Flink.groovy | 16 +- .../job_PostCommit_Java_Nexmark_Spark.groovy | 8 +- ...PortableValidatesRunner_Flink_Batch.groovy | 2 +- ...ableValidatesRunner_Flink_Streaming.groovy | 2 +- ...ostCommit_Java_ValidatesRunner_Apex.groovy | 2 +- ...ommit_Java_ValidatesRunner_Dataflow.groovy | 2 +- ..._DataflowPortabilityExecutableStage.groovy | 2 +- ...ava_ValidatesRunner_Dataflow_Java11.groovy | 2 +- ..._Java_ValidatesRunner_Direct_Java11.groovy | 6 +- ...stCommit_Java_ValidatesRunner_Flink.groovy | 2 +- ...ommit_Java_ValidatesRunner_Gearpump.groovy | 2 +- ...datesRunner_PortabilityApi_Dataflow.groovy | 2 +- ...stCommit_Java_ValidatesRunner_Samza.groovy | 2 +- ...stCommit_Java_ValidatesRunner_Spark.groovy | 2 +- ...mit_Python_ValidatesRunner_Dataflow.groovy | 4 +- .../job_PostCommit_Website_Publish.groovy | 4 +- ...Commit_Python_ValidatesRunner_Flink.groovy | 2 +- .../job_PreCommit_Website_Stage_GCS.groovy | 2 +- .../job_Release_Python_NightlySnapshot.groovy | 4 +- build.gradle | 84 ++--- .../beam/gradle/BeamModulePlugin.groovy | 53 +-- examples/java/build.gradle | 24 +- examples/kotlin/build.gradle | 22 +- model/fn-execution/build.gradle | 4 +- model/job-management/build.gradle | 4 +- release/build.gradle | 14 +- .../main/scripts/build_release_candidate.sh | 2 +- release/src/main/scripts/run_rc_validation.sh | 12 +- runners/apex/build.gradle | 22 +- runners/core-construction-java/build.gradle | 8 +- runners/core-java/build.gradle | 12 +- .../metrics/MonitoringInfoMetricName.java | 2 +- runners/direct-java/build.gradle | 64 ++-- runners/extensions-java/metrics/build.gradle | 2 +- runners/flink/1.5/build.gradle | 1 + runners/flink/1.5/job-server/build.gradle | 1 + runners/flink/1.6/build.gradle | 1 + runners/flink/1.6/job-server/build.gradle | 1 + runners/flink/1.7/build.gradle | 1 + runners/flink/1.7/job-server/build.gradle | 1 + runners/flink/1.8/build.gradle | 1 + runners/flink/1.8/job-server/build.gradle | 1 + runners/flink/flink_runner.gradle | 49 +-- .../flink_job_server_container.gradle | 2 +- .../flink/job-server/flink_job_server.gradle | 19 +- runners/gearpump/build.gradle | 16 +- .../google-cloud-dataflow-java/build.gradle | 92 ++--- .../examples-streaming/build.gradle | 24 +- .../examples/build.gradle | 38 +- .../worker/build.gradle | 34 +- .../worker/legacy-worker/build.gradle | 54 +-- .../worker/windmill/build.gradle | 1 + runners/java-fn-execution/build.gradle | 24 +- runners/local-java/build.gradle | 5 +- runners/reference/java/build.gradle | 12 +- runners/reference/job-server/build.gradle | 4 +- runners/samza/build.gradle | 22 +- runners/samza/job-server/build.gradle | 2 +- runners/spark/build.gradle | 32 +- runners/spark/job-server/build.gradle | 10 +- sdks/go/README.md | 2 +- sdks/go/container/build.gradle | 6 +- sdks/go/examples/build.gradle | 6 +- sdks/go/test/build.gradle | 6 +- sdks/go/test/run_integration_tests.sh | 2 +- sdks/java/bom/build.gradle | 10 +- sdks/java/container/build.gradle | 10 +- sdks/java/core/build.gradle | 4 +- sdks/java/extensions/euphoria/build.gradle | 8 +- .../google-cloud-platform-core/build.gradle | 4 +- sdks/java/extensions/jackson/build.gradle | 5 +- .../java/extensions/join-library/build.gradle | 4 +- sdks/java/extensions/kryo/build.gradle | 6 +- sdks/java/extensions/protobuf/build.gradle | 4 +- sdks/java/extensions/sketching/build.gradle | 6 +- sdks/java/extensions/sorter/build.gradle | 4 +- sdks/java/extensions/sql/build.gradle | 14 +- .../extensions/sql/datacatalog/build.gradle | 4 +- .../java/extensions/sql/hcatalog/build.gradle | 4 +- sdks/java/extensions/sql/jdbc/build.gradle | 10 +- sdks/java/extensions/sql/shell/build.gradle | 6 +- .../sql/example/BeamSqlExample.java | 2 +- .../sql/example/BeamSqlPojoExample.java | 2 +- sdks/java/fn-execution/build.gradle | 6 +- sdks/java/harness/build.gradle | 10 +- sdks/java/io/amazon-web-services/build.gradle | 8 +- sdks/java/io/amqp/build.gradle | 4 +- sdks/java/io/cassandra/build.gradle | 6 +- sdks/java/io/clickhouse/build.gradle | 4 +- sdks/java/io/common/build.gradle | 2 +- .../elasticsearch-tests-2/build.gradle | 11 +- .../elasticsearch-tests-5/build.gradle | 11 +- .../elasticsearch-tests-6/build.gradle | 11 +- .../elasticsearch-tests-common/build.gradle | 9 +- sdks/java/io/elasticsearch/build.gradle | 4 +- sdks/java/io/file-based-io-tests/build.gradle | 10 +- .../io/google-cloud-platform/build.gradle | 12 +- .../beam/sdk/io/gcp/storage/GcsKmsKeyIT.java | 2 +- sdks/java/io/hadoop-common/build.gradle | 2 +- sdks/java/io/hadoop-file-system/build.gradle | 8 +- sdks/java/io/hadoop-format/build.gradle | 26 +- sdks/java/io/hbase/build.gradle | 10 +- sdks/java/io/hcatalog/build.gradle | 10 +- sdks/java/io/jdbc/build.gradle | 10 +- sdks/java/io/jms/build.gradle | 4 +- sdks/java/io/kafka/build.gradle | 8 +- sdks/java/io/kinesis/build.gradle | 4 +- sdks/java/io/kudu/build.gradle | 8 +- sdks/java/io/mongodb/build.gradle | 8 +- sdks/java/io/mqtt/build.gradle | 6 +- sdks/java/io/parquet/build.gradle | 6 +- sdks/java/io/rabbitmq/build.gradle | 7 +- sdks/java/io/redis/build.gradle | 6 +- sdks/java/io/solr/build.gradle | 8 +- sdks/java/io/synthetic/build.gradle | 4 +- sdks/java/io/tika/build.gradle | 6 +- sdks/java/io/xml/build.gradle | 6 +- sdks/java/javadoc/build.gradle | 12 +- .../maven-archetypes/examples/build.gradle | 8 +- .../maven-archetypes/starter/build.gradle | 4 +- sdks/java/testing/load-tests/build.gradle | 29 +- .../sdk/loadtests/CoGroupByKeyLoadTest.java | 2 +- .../beam/sdk/loadtests/CombineLoadTest.java | 2 +- .../sdk/loadtests/GroupByKeyLoadTest.java | 2 +- .../beam/sdk/loadtests/ParDoLoadTest.java | 2 +- .../sdk/loadtests/SyntheticDataPublisher.java | 2 +- sdks/java/testing/nexmark/build.gradle | 39 +- sdks/java/testing/test-utils/build.gradle | 7 +- .../io/gcp/gcsio_integration_test.py | 2 +- .../load_tests/co_group_by_key_test.py | 4 +- .../testing/load_tests/combine_test.py | 4 +- .../testing/load_tests/group_by_key_test.py | 4 +- .../testing/load_tests/pardo_test.py | 4 +- .../testing/load_tests/sideinput_test.py | 4 +- sdks/python/build.gradle | 64 ++-- sdks/python/container/build.gradle | 8 +- sdks/python/container/py3/build.gradle | 4 +- .../container/run_validatescontainer.sh | 4 +- sdks/python/scripts/run_integration_test.sh | 2 +- sdks/python/scripts/run_snapshot_publish.sh | 2 +- .../test-suites/dataflow/py35/build.gradle | 8 +- .../test-suites/dataflow/py36/build.gradle | 8 +- settings.gradle | 342 ++++++------------ .../build.gradle | 6 +- website/README.md | 4 +- website/_config_test.yml | 2 +- website/build.gradle | 2 +- website/src/contribute/release-guide.md | 12 +- website/src/documentation/dsls/sql/shell.md | 6 +- website/src/documentation/runners/flink.md | 6 +- website/src/documentation/sdks/euphoria.md | 4 +- website/src/documentation/sdks/nexmark.md | 40 +- website/src/roadmap/portability.md | 8 +- 159 files changed, 931 insertions(+), 1011 deletions(-) diff --git a/.test-infra/jenkins/CommonTestProperties.groovy b/.test-infra/jenkins/CommonTestProperties.groovy index 2d8d59654d95f..ee4eddc877279 100644 --- a/.test-infra/jenkins/CommonTestProperties.groovy +++ b/.test-infra/jenkins/CommonTestProperties.groovy @@ -32,10 +32,10 @@ class CommonTestProperties { def RUNNER_DEPENDENCY_MAP = [ JAVA: [ - DATAFLOW: ":beam-runners-google-cloud-dataflow-java", - SPARK: ":beam-runners-spark", - FLINK: ":beam-runners-flink_2.11", - DIRECT: ":beam-runners-direct-java" + DATAFLOW: ":runners:google-cloud-dataflow-java", + SPARK: ":runners:spark", + FLINK: ":runners:flink:1.5", + DIRECT: ":runners:direct-java" ], PYTHON: [ DATAFLOW: "TestDataflowRunner", diff --git a/.test-infra/jenkins/LoadTestsBuilder.groovy b/.test-infra/jenkins/LoadTestsBuilder.groovy index 08bbb30b6f1e7..517046533f1c4 100644 --- a/.test-infra/jenkins/LoadTestsBuilder.groovy +++ b/.test-infra/jenkins/LoadTestsBuilder.groovy @@ -48,9 +48,9 @@ class LoadTestsBuilder { private static String getGradleTaskName(SDK sdk) { if (sdk == SDK.JAVA) { - return ':beam-sdks-java-load-tests:run' + return ':beam:sdks:java:load-tests:run' } else if (sdk == SDK.PYTHON) { - return ':beam-sdks-python-load-tests:run' + return ':sdks:python:apache_beam:testing:load-tests:run' } else { throw new RuntimeException("No task name defined for SDK: $SDK") } diff --git a/.test-infra/jenkins/NexmarkBuilder.groovy b/.test-infra/jenkins/NexmarkBuilder.groovy index f2b0c2473fe13..32a4e13136d0e 100644 --- a/.test-infra/jenkins/NexmarkBuilder.groovy +++ b/.test-infra/jenkins/NexmarkBuilder.groovy @@ -79,7 +79,7 @@ class NexmarkBuilder { shell("echo *** RUN ${title} ***") gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) switches("-Pnexmark.runner=${runner.getDepenedencyBySDK(sdk)}") switches("-Pnexmark.args=\"${parseOptions(options)}\"") diff --git a/.test-infra/jenkins/job_PostCommit_Java11_ValidatesRunner_PortabilityApi_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java11_ValidatesRunner_PortabilityApi_Dataflow.groovy index 9210139dd65f0..fa5053a99a1cd 100644 --- a/.test-infra/jenkins/job_PostCommit_Java11_ValidatesRunner_PortabilityApi_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java11_ValidatesRunner_PortabilityApi_Dataflow.groovy @@ -34,7 +34,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_ValidatesRunner_Porta steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-google-cloud-dataflow-java:validatesRunnerFnApiWorkerTest') + tasks(':runners:google-cloud-dataflow-java:validatesRunnerFnApiWorkerTest') switches ('-Pdockerfile=Dockerfile-java11') // Increase parallel worker threads above processor limit since most time is diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow.groovy index 30b59f1fba44b..2b6f3c35fe946 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow.groovy @@ -38,9 +38,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN BATCH MODE USING DATAFLOW RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-google-cloud-dataflow-java"' + + switches('-Pnexmark.runner=":runners:google-cloud-dataflow-java"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--runner=DataflowRunner', @@ -58,9 +58,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN STREAMING MODE USING DATAFLOW RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-google-cloud-dataflow-java"' + + switches('-Pnexmark.runner=":runners:google-cloud-dataflow-java"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--runner=DataflowRunner', @@ -78,9 +78,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN SQL BATCH MODE USING DATAFLOW RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-google-cloud-dataflow-java"' + + switches('-Pnexmark.runner=":runners:google-cloud-dataflow-java"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--runner=DataflowRunner', @@ -99,9 +99,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN SQL STREAMING MODE USING DATAFLOW RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-google-cloud-dataflow-java"' + + switches('-Pnexmark.runner=":runners:google-cloud-dataflow-java"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--runner=DataflowRunner', diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy index 131805971dd55..706b536a4863c 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy @@ -37,9 +37,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN BATCH MODE USING DIRECT RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-direct-java"' + + switches('-Pnexmark.runner=":runners:direct-java"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--runner=DirectRunner', @@ -53,9 +53,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN STREAMING MODE USING DIRECT RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-direct-java"' + + switches('-Pnexmark.runner=":runners:direct-java"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--runner=DirectRunner', @@ -69,9 +69,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN SQL BATCH MODE USING DIRECT RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-direct-java"' + + switches('-Pnexmark.runner=":runners:direct-java"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--runner=DirectRunner', @@ -86,9 +86,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN SQL STREAMING MODE USING DIRECT RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-direct-java"' + + switches('-Pnexmark.runner=":runners:direct-java"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--runner=DirectRunner', diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy index 63fc48c31b50e..065f74d57f3c9 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy @@ -38,9 +38,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN BATCH MODE USING FLINK RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-flink_2.11"' + + switches('-Pnexmark.runner=":runners:flink:1.5"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--streaming=false', @@ -53,9 +53,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN STREAMING MODE USING FLINK RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-flink_2.11"' + + switches('-Pnexmark.runner=":runners:flink:1.5"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--streaming=true', @@ -68,9 +68,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN SQL BATCH MODE USING FLINK RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-flink_2.11"' + + switches('-Pnexmark.runner=":runners:flink:1.5"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--queryLanguage=sql', @@ -83,9 +83,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN SQL STREAMING MODE USING FLINK RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-flink_2.11"' + + switches('-Pnexmark.runner=":runners:flink:1.5"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--queryLanguage=sql', diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy index 20847dc80c051..5fb7d24051ba2 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy @@ -36,9 +36,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK IN BATCH MODE USING SPARK RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-spark"' + + switches('-Pnexmark.runner=":runners:spark"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--runner=SparkRunner', @@ -51,9 +51,9 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_ shell('echo *** RUN NEXMARK SQL IN BATCH MODE USING SPARK RUNNER ***') gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-java-nexmark:run') + tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches('-Pnexmark.runner=":beam-runners-spark"' + + switches('-Pnexmark.runner=":runners:spark"' + ' -Pnexmark.args="' + [NexmarkBigqueryProperties.nexmarkBigQueryArgs, '--runner=SparkRunner', diff --git a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy index 7aee97a20f2aa..b8a59b3702c9b 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy @@ -36,7 +36,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Flink_Batch', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-flink_2.11-job-server:validatesPortableRunnerBatch') + tasks(':runners:flink:1.5:job-server:validatesPortableRunnerBatch') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy index c016a5fff5c53..6a48e31a3c851 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy @@ -36,7 +36,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Flink_Streaming', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-flink_2.11-job-server:validatesPortableRunnerStreaming') + tasks(':runners:flink:1.5:job-server:validatesPortableRunnerStreaming') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Apex.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Apex.groovy index faaa7af9aa0d5..3cbf3966afec9 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Apex.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Apex.groovy @@ -37,7 +37,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Apex', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-apex:validatesRunner') + tasks(':runners:apex:validatesRunner') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy index 72802896d94d5..530fba65b3a47 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy @@ -40,7 +40,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflo steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-google-cloud-dataflow-java:validatesRunner') + tasks(':runners:google-cloud-dataflow-java:validatesRunner') // Increase parallel worker threads above processor limit since most time is // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow // because each one launches a Dataflow job with about 3 mins of overhead. diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_DataflowPortabilityExecutableStage.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_DataflowPortabilityExecutableStage.groovy index 62e73617b74ef..3826c7831c67e 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_DataflowPortabilityExecutableStage.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_DataflowPortabilityExecutableStage.groovy @@ -39,7 +39,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflo steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-google-cloud-dataflow-java:validatesRunnerFnApiWorkerExecutableStageTest') + tasks(':runners:google-cloud-dataflow-java:validatesRunnerFnApiWorkerExecutableStageTest') // Increase parallel worker threads above processor limit since most time is // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow // because each one launches a Dataflow job with about 3 mins of overhead. diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy index 4e6da6a1e6ff7..74e49b62b398f 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy @@ -34,7 +34,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_ValidatesRunner_Dataf steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-google-cloud-dataflow-java:validatesJava11Runner') + tasks(':runners:google-cloud-dataflow-java:validatesJava11Runner') // Increase parallel worker threads above processor limit since most time is // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow // because each one launches a Dataflow job with about 3 mins of overhead. diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct_Java11.groovy index 24d925741da6a..170ae046be0f9 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct_Java11.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct_Java11.groovy @@ -38,14 +38,14 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_ValidatesRunner_Direc steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-direct-java:shadowJar') - tasks(':beam-runners-direct-java:shadowTestJar') + tasks(':runners:direct-java:shadowJar') + tasks(':runners:direct-java:shadowTestJar') switches("-Dorg.gradle.java.home=${JAVA_8_HOME}") } gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-direct-java:validatesRunner') + tasks(':runners:direct-java:validatesRunner') switches("-Dorg.gradle.java.home=${JAVA_11_HOME}") switches('-x shadowJar') switches('-x shadowTestJar') diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy index d21ae86b64f69..2c8b21269ad0a 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy @@ -37,7 +37,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Flink', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-flink_2.11:validatesRunner') + tasks(':runners:flink:1.5:validatesRunner') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Gearpump.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Gearpump.groovy index 2b0dd7d36e4e2..e1ef58ff8c5b2 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Gearpump.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Gearpump.groovy @@ -41,7 +41,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Gearpum steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-gearpump:validatesRunner') + tasks(':runners:gearpump:validatesRunner') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_PortabilityApi_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_PortabilityApi_Dataflow.groovy index 587e8850f977b..54ad76435b57a 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_PortabilityApi_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_PortabilityApi_Dataflow.groovy @@ -40,7 +40,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Portabi steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-google-cloud-dataflow-java:validatesRunnerFnApiWorkerTest') + tasks(':runners:google-cloud-dataflow-java:validatesRunnerFnApiWorkerTest') // Increase parallel worker threads above processor limit since most time is // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow // because each one launches a Dataflow job with about 3 mins of overhead. diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Samza.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Samza.groovy index fe08abd6d6f3f..dd20d1b813b40 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Samza.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Samza.groovy @@ -37,7 +37,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Samza', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-samza:validatesRunner') + tasks(':unners:samza:validatesRunner') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy index 8804d0df70cbd..6dc0fcf159ee0 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy @@ -37,7 +37,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Spark', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-runners-spark:validatesRunner') + tasks(':runners:spark:validatesRunner') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy index cfdd01bf99665..aa8d89ae6522a 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy @@ -32,8 +32,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow', 'Run Python steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-python:validatesRunnerBatchTests') - tasks(':beam-sdks-python:validatesRunnerStreamingTests') + tasks(':sdks:python:validatesRunnerBatchTests') + tasks(':sdks:python:validatesRunnerStreamingTests') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy b/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy index 669a84abcfeee..55fad496d15fb 100644 --- a/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy +++ b/.test-infra/jenkins/job_PostCommit_Website_Publish.groovy @@ -33,8 +33,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Website_Publish', '', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-website:clean') - tasks(':beam-website:publishWebsite') + tasks(':website:clean') + tasks(':website:publishWebsite') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy index 0113f7adc8055..086578c7d10cc 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy @@ -23,7 +23,7 @@ import PrecommitJobBuilder PrecommitJobBuilder builder = new PrecommitJobBuilder( scope: this, nameBase: 'Python_PVR_Flink', - gradleTask: ':beam-sdks-python:flinkValidatesRunner', + gradleTask: ':sdks:python:flinkValidatesRunner', triggerPathPatterns: [ '^model/.*$', '^runners/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy b/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy index c5cf28f2efa6b..03b6c346b105c 100644 --- a/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy +++ b/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy @@ -21,7 +21,7 @@ import PrecommitJobBuilder PrecommitJobBuilder builder = new PrecommitJobBuilder( scope: this, nameBase: 'Website_Stage_GCS', - gradleTask: ':beam-website:stageWebsite', + gradleTask: ':website:stageWebsite', triggerPathPatterns: ['^website/.*$'] ) builder.build { diff --git a/.test-infra/jenkins/job_Release_Python_NightlySnapshot.groovy b/.test-infra/jenkins/job_Release_Python_NightlySnapshot.groovy index ab02b3a208469..8548b150f614f 100644 --- a/.test-infra/jenkins/job_Release_Python_NightlySnapshot.groovy +++ b/.test-infra/jenkins/job_Release_Python_NightlySnapshot.groovy @@ -44,13 +44,13 @@ job('beam_Release_Python_NightlySnapshot') { // Cleanup Python directory. gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-python:clean') + tasks(':sdks:python:clean') commonJobProperties.setGradleSwitches(delegate) } // Build snapshot. gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-sdks-python:buildSnapshot') + tasks(':sdks:python:buildSnapshot') commonJobProperties.setGradleSwitches(delegate) } // Publish snapshot to a public accessible GCS directory. diff --git a/build.gradle b/build.gradle index 0b2128a79998f..68201106bacbb 100644 --- a/build.gradle +++ b/build.gradle @@ -125,48 +125,48 @@ check.dependsOn rat task javaPreCommit() { // We need to list the model/* builds since sdks/java/core doesn't // depend on any of the model. - dependsOn ":beam-model-pipeline:build" - dependsOn ":beam-model-job-management:build" - dependsOn ":beam-model-fn-execution:build" - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:build" - dependsOn ":beam-sdks-java-core:buildNeeded" - dependsOn ":beam-sdks-java-core:buildDependents" - dependsOn ":beam-examples-java:preCommit" - dependsOn ":beam-sdks-java-extensions-sql-jdbc:preCommit" - dependsOn ":beam-sdks-java-javadoc:allJavadoc" - dependsOn ":beam-runners-direct-java:needsRunnerTests" + dependsOn ":model:pipeline:build" + dependsOn ":model:job-management:build" + dependsOn ":model:fn-execution:build" + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:build" + dependsOn ":sdks:java:core:buildNeeded" + dependsOn ":sdks:java:core:buildDependents" + dependsOn ":examples:java:preCommit" + dependsOn ":sdks:java:extensions:sql:jdbc:preCommit" + dependsOn ":sdks:java:javadoc:allJavadoc" + dependsOn ":runners:direct-java:needsRunnerTests" } task javaPreCommitPortabilityApi() { - dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:build" - dependsOn ":beam-runners-google-cloud-dataflow-java-examples:verifyPortabilityApi" + dependsOn ":runners:google-cloud-dataflow-java:worker:build" + dependsOn ":runners:google-cloud-dataflow-java:examples:verifyPortabilityApi" } task javaPostCommit() { - dependsOn ":beam-runners-google-cloud-dataflow-java:postCommit" - dependsOn ":beam-sdks-java-extensions-google-cloud-platform-core:postCommit" - dependsOn ":beam-sdks-java-io-google-cloud-platform:postCommit" + dependsOn ":runners:google-cloud-dataflow-java:postCommit" + dependsOn ":sdks:java:extensions:google-cloud-platform-core:postCommit" + dependsOn ":sdks:java:io:google-cloud-platform:postCommit" } task sqlPostCommit() { - dependsOn ":beam-sdks-java-extensions-sql:postCommit" - dependsOn ":beam-sdks-java-extensions-sql-jdbc:postCommit" + dependsOn ":sdks:java:extensions:sql:postCommit" + dependsOn ":sdks:java:extensions:sql:jdbc:postCommit" } task javaPostCommitPortabilityApi () { - dependsOn ":beam-runners-google-cloud-dataflow-java:postCommitPortabilityApi" + dependsOn ":runners:google-cloud-dataflow-java:postCommitPortabilityApi" } task goPreCommit() { - dependsOn ":beam-sdks-go:goTest" + dependsOn ":sdks:go:goTest" - dependsOn ":beam-sdks-go-examples:build" - dependsOn ":beam-sdks-go-test:build" + dependsOn ":sdks:go:examples:build" + dependsOn ":sdks:go:test:build" // Ensure all container Go boot code builds as well. - dependsOn ":beam-sdks-java-container:build" - dependsOn ":beam-sdks-python-container:build" - dependsOn ":beam-sdks-go-container:build" + dependsOn ":sdks:java:container:build" + dependsOn ":sdks:python:container:build" + dependsOn ":sdks:go:container:build" } task goPostCommit() { @@ -180,36 +180,36 @@ task goIntegrationTests() { args '-c', './sdks/go/test/run_integration_tests.sh' } } - dependsOn ":beam-sdks-go-test:build" - dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" + dependsOn ":sdks:go:test:build" + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" } task pythonPreCommit() { - dependsOn ":beam-sdks-python:preCommitPy2" - dependsOn ":beam-sdks-python-test-suites-tox-py35:preCommitPy35" - dependsOn ":beam-sdks-python-test-suites-tox-py36:preCommitPy36" - dependsOn ":beam-sdks-python-test-suites-tox-py37:preCommitPy37" - dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT" + dependsOn ":sdks:python:preCommitPy2" + dependsOn ":sdks:python:test-suites:tox:py35:preCommitPy35" + dependsOn ":sdks:python:test-suites:tox:py36:preCommitPy36" + dependsOn ":sdks:python:test-suites:tox:py37:preCommitPy37" + dependsOn ":sdks:python:test-suites:dataflow:preCommitIT" } task pythonPostCommit() { - dependsOn ":beam-sdks-python:postCommit" + dependsOn ":sdks:python:postCommit" } task python3PostCommit() { - dependsOn ":beam-sdks-python-test-suites-dataflow-py35:postCommitIT" - dependsOn ":beam-sdks-python-test-suites-dataflow-py36:postCommitIT" - dependsOn ":beam-sdks-python-test-suites-dataflow-py35:validatesRunnerBatchTests" - dependsOn ":beam-sdks-python-test-suites-direct-py35:postCommitIT" - dependsOn ":beam-sdks-python-test-suites-direct-py36:postCommitIT" + dependsOn ":sdks:python:test-suites:dataflow:py35:postCommitIT" + dependsOn ":sdks:python:test-suites:dataflow:py36:postCommitIT" + dependsOn ":sdks:python:test-suites:dataflow:py35:validatesRunnerBatchTests" + dependsOn ":sdks:python:test-suites:direct:py35:postCommitIT" + dependsOn ":sdks:python:test-suites:direct:py36:postCommitIT" } task portablePythonPreCommit() { - dependsOn ":beam-sdks-python:portablePreCommit" + dependsOn ":sdks:python:portablePreCommit" } task websitePreCommit() { - dependsOn ":beam-website:preCommit" + dependsOn ":website:preCommit" } task communityMetricsPreCommit() { @@ -221,13 +221,13 @@ task communityMetricsProber() { } task javaExamplesDataflowPrecommit() { - dependsOn ":beam-runners-google-cloud-dataflow-java-examples:preCommit" - dependsOn ":beam-runners-google-cloud-dataflow-java-examples-streaming:preCommit" + dependsOn ":runners:google-cloud-dataflow-java:examples:preCommit" + dependsOn ":runners:google-cloud-dataflow-java:examples-streaming:preCommit" } task runBeamDependencyCheck() { dependsOn ":dependencyUpdates" - dependsOn ":beam-sdks-python:dependencyUpdates" + dependsOn ":sdks:python:dependencyUpdates" } // Configure the release plugin to do only local work; the release manager determines what, if diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index f669f02732358..92541102e9b74 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -302,7 +302,7 @@ class BeamModulePlugin implements Plugin { /** ***********************************************************************************************/ // Apply common properties/repositories and tasks to all projects. - project.group = 'org.apache.beam' + project.ext.mavenGroupId = 'org.apache.beam' // Automatically use the official release version if we are performing a release // otherwise append '-SNAPSHOT' @@ -311,6 +311,9 @@ class BeamModulePlugin implements Plugin { project.version += '-SNAPSHOT' } + project.apply plugin: 'base' + project.archivesBaseName = 'beam' + project.path.replace(':', '-') + project.apply plugin: 'org.apache.beam.jenkins' // Register all Beam repositories and configuration tweaks @@ -958,8 +961,8 @@ class BeamModulePlugin implements Plugin { doLast { new File("${pomPropertiesFile}").text = """version=${project.version} - groupId=${project.group} - artifactId=${project.name}""" + groupId=${project.mavenGroupId} + artifactId=${project.archivesBaseName}""" } } @@ -979,13 +982,13 @@ class BeamModulePlugin implements Plugin { } dependsOn 'generatePomFileForMavenJavaPublication' - into("META-INF/maven/${project.group}/${project.name}") { + into("META-INF/maven/${project.mavenGroupId}/${project.archivesBaseName}") { from "${pomFile}" rename('.*', 'pom.xml') } dependsOn project.generatePomPropertiesFileForMavenJavaPublication - into("META-INF/maven/${project.group}/${project.name}") { from "${pomPropertiesFile}" } + into("META-INF/maven/${project.mavenGroupId}/${project.archivesBaseName}") { from "${pomPropertiesFile}" } } // Only build artifacts for archives if we are publishing @@ -1021,6 +1024,9 @@ class BeamModulePlugin implements Plugin { artifact project.testSourcesJar artifact project.javadocJar + artifactId = project.archivesBaseName + groupId = project.mavenGroupId + pom { name = project.description if (project.hasProperty("summary")) { @@ -1198,10 +1204,10 @@ class BeamModulePlugin implements Plugin { def pipelineOptionsString = configuration.integrationTestPipelineOptions if(pipelineOptionsString && configuration.runner?.equalsIgnoreCase('dataflow')) { - project.evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker") + project.evaluationDependsOn(":runners:google-cloud-dataflow-java:worker:legacy-worker") def allOptionsList = (new JsonSlurper()).parseText(pipelineOptionsString) def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: - project.project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath + project.project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath allOptionsList.addAll([ '--workerHarnessContainerImage=', @@ -1230,20 +1236,20 @@ class BeamModulePlugin implements Plugin { /* include dependencies required by runners */ //if (runner?.contains('dataflow')) { if (runner?.equalsIgnoreCase('dataflow')) { - testCompile it.project(path: ":beam-runners-google-cloud-dataflow-java", configuration: 'shadowTest') - shadow it.project(path: ":beam-runners-google-cloud-dataflow-java-legacy-worker", configuration: 'shadow') + testCompile it.project(path: ":runners:google-cloud-dataflow-java", configuration: 'shadowTest') + shadow it.project(path: ":runners:google-cloud-dataflow-java:worker:legacy-worker", configuration: 'shadow') } if (runner?.equalsIgnoreCase('direct')) { - testCompile it.project(path: ":beam-runners-direct-java", configuration: 'shadowTest') + testCompile it.project(path: ":runners:direct-java", configuration: 'shadowTest') } if (runner?.equalsIgnoreCase('flink')) { - testCompile it.project(path: ":beam-runners-flink_2.11", configuration: 'shadowTest') + testCompile it.project(path: ":runners:flink:1.5", configuration: 'shadowTest') } if (runner?.equalsIgnoreCase('spark')) { - testCompile it.project(path: ":beam-runners-spark", configuration: 'shadowTest') + testCompile it.project(path: ":runners:spark", configuration: 'shadowTest') testCompile project.library.java.spark_core testCompile project.library.java.spark_streaming @@ -1255,13 +1261,13 @@ class BeamModulePlugin implements Plugin { /* include dependencies required by filesystems */ if (filesystem?.equalsIgnoreCase('hdfs')) { - testCompile it.project(path: ":beam-sdks-java-io-hadoop-file-system", configuration: 'shadowTest') + testCompile it.project(path: ":sdks:java:io:hadoop-file-system", configuration: 'shadowTest') shadowTest project.library.java.hadoop_client } /* include dependencies required by AWS S3 */ if (filesystem?.equalsIgnoreCase('s3')) { - testCompile it.project(path: ":beam-sdks-java-io-amazon-web-services", configuration: 'shadowTest') + testCompile it.project(path: ":sdks:java:io:amazon-web-services", configuration: 'shadowTest') } } @@ -1570,9 +1576,9 @@ class BeamModulePlugin implements Plugin { * the following projects are evaluated before we evaluate this project. This is because * we are attempting to reference the "sourceSets.test.output" directly. */ - project.evaluationDependsOn(":beam-sdks-java-core") - project.evaluationDependsOn(":beam-runners-core-java") - project.evaluationDependsOn(":beam-runners-core-construction-java") + project.evaluationDependsOn(":sdks:java:core") + project.evaluationDependsOn(":runners:core-java") + project.evaluationDependsOn(":runners:core-construction-java") def config = it ? it as PortableValidatesRunnerConfiguration : new PortableValidatesRunnerConfiguration() def name = config.name def beamTestPipelineOptions = [ @@ -1595,14 +1601,14 @@ class BeamModulePlugin implements Plugin { description = "Validates the PortableRunner with JobServer ${config.jobServerDriver}" systemProperties config.systemProperties classpath = config.testClasspathConfiguration - testClassesDirs = project.files(project.project(":beam-sdks-java-core").sourceSets.test.output.classesDirs, project.project(":beam-runners-core-java").sourceSets.test.output.classesDirs, project.project(":beam-runners-core-construction-java").sourceSets.test.output.classesDirs) + testClassesDirs = project.files(project.project(":sdks:java:core").sourceSets.test.output.classesDirs, project.project(":runners:core-java").sourceSets.test.output.classesDirs) maxParallelForks config.numParallelTests useJUnit(config.testCategories) // increase maxHeapSize as this is directly correlated to direct memory, // see https://issues.apache.org/jira/browse/BEAM-6698 maxHeapSize = '4g' if (config.environment == PortableValidatesRunnerConfiguration.Environment.DOCKER) { - dependsOn ':beam-sdks-java-container:docker' + dependsOn ':sdks:java:container:docker' } } } @@ -1620,20 +1626,19 @@ class BeamModulePlugin implements Plugin { project.task('test') {} project.check.dependsOn project.test - project.evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-fn-api-worker") + project.evaluationDependsOn(":runners:google-cloud-dataflow-java:worker") // Due to Beam-4256, we need to limit the length of virtualenv path to make the // virtualenv activated properly. So instead of include project name in the path, // we use the hash value. - project.ext.envdir = "${project.rootProject.buildDir}/gradleenv/${project.name.hashCode()}" + project.ext.envdir = "${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}" def pythonRootDir = "${project.rootDir}/sdks/python" // Python interpreter version for virtualenv setup and test run. This value can be // set from commandline with -PpythonVersion, or in build script of certain project. // If none of them applied, version set here will be used as default value. - if(!project.hasProperty('pythonVersion')) { - project.ext.pythonVersion = '2.7' - } + project.ext.pythonVersion = project.hasProperty('pythonVersion') ? + project.pythonVersion : '2.7' project.task('setupVirtualenv') { doLast { diff --git a/examples/java/build.gradle b/examples/java/build.gradle index fc46a2d51660e..1f2059d160ee1 100644 --- a/examples/java/build.gradle +++ b/examples/java/build.gradle @@ -46,9 +46,9 @@ configurations.sparkRunnerPreCommit { dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadow") + shadow project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow") shadow library.java.google_api_client shadow library.java.google_api_services_bigquery shadow library.java.google_http_client @@ -62,8 +62,8 @@ dependencies { shadow library.java.joda_time shadow library.java.slf4j_api shadow library.java.slf4j_jdk14 - runtime project(path: ":beam-runners-direct-java", configuration: "shadow") - shadowTest project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") + runtime project(path: ":runners:direct-java", configuration: "shadow") + shadowTest project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow") shadowTest library.java.hamcrest_core shadowTest library.java.hamcrest_library shadowTest library.java.junit @@ -72,18 +72,18 @@ dependencies { // Add dependencies for the PreCommit configurations // For each runner a project level dependency on the examples project. for (String runner : preCommitRunners) { - delegate.add(runner + "PreCommit", project(path: ":beam-examples-java", configuration: "shadow")) - delegate.add(runner + "PreCommit", project(path: ":beam-examples-java", configuration: "shadowTest")) + delegate.add(runner + "PreCommit", project(path: ":examples:java", configuration: "shadow")) + delegate.add(runner + "PreCommit", project(path: ":examples:java", configuration: "shadowTest")) } // https://issues.apache.org/jira/browse/BEAM-3583 - // apexRunnerPreCommit project(path: ":beam-runners-apex", configuration: "shadow") - directRunnerPreCommit project(path: ":beam-runners-direct-java", configuration: "shadow") - flinkRunnerPreCommit project(path: ":beam-runners-flink_2.11", configuration: "shadow") + // apexRunnerPreCommit project(path: ":runners:apex", configuration: "shadow") + directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow") + flinkRunnerPreCommit project(path: ":runners:flink:1.5", configuration: "shadow") // TODO: Make the netty version used configurable, we add netty-all 4.1.17.Final so it appears on the classpath // before 4.1.8.Final defined by Apache Beam sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final" - sparkRunnerPreCommit project(path: ":beam-runners-spark", configuration: "shadow") - sparkRunnerPreCommit project(path: ":beam-sdks-java-io-hadoop-file-system", configuration: "shadow") + sparkRunnerPreCommit project(path: ":runners:spark", configuration: "shadow") + sparkRunnerPreCommit project(path: ":sdks:java:io:hadoop-file-system", configuration: "shadow") sparkRunnerPreCommit library.java.spark_streaming sparkRunnerPreCommit library.java.spark_core } diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle index 9d417e3d59ac9..ff54658eb3a91 100644 --- a/examples/kotlin/build.gradle +++ b/examples/kotlin/build.gradle @@ -49,9 +49,9 @@ configurations.sparkRunnerPreCommit { dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadow") + shadow project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow") shadow library.java.google_api_client shadow library.java.google_api_services_bigquery shadow library.java.google_http_client @@ -65,8 +65,8 @@ dependencies { shadow library.java.joda_time shadow library.java.slf4j_api shadow library.java.slf4j_jdk14 - runtime project(path: ":beam-runners-direct-java", configuration: "shadow") - shadowTest project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") + runtime project(path: ":runners:direct-java", configuration: "shadow") + shadowTest project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow") shadowTest library.java.hamcrest_core shadowTest library.java.hamcrest_library shadowTest library.java.junit @@ -75,18 +75,18 @@ dependencies { // Add dependencies for the PreCommit configurations // For each runner a project level dependency on the examples project. for (String runner : preCommitRunners) { - delegate.add(runner + "PreCommit", project(path: ":beam-examples-kotlin", configuration: "shadow")) - delegate.add(runner + "PreCommit", project(path: ":beam-examples-kotlin", configuration: "shadowTest")) + delegate.add(runner + "PreCommit", project(path: ":examples:kotlin", configuration: "shadow")) + delegate.add(runner + "PreCommit", project(path: ":examples:kotlin", configuration: "shadowTest")) } // https://issues.apache.org/jira/browse/BEAM-3583 // apexRunnerPreCommit project(path: ":beam-runners-apex", configuration: "shadow") - directRunnerPreCommit project(path: ":beam-runners-direct-java", configuration: "shadow") - flinkRunnerPreCommit project(path: ":beam-runners-flink_2.11", configuration: "shadow") + directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow") + flinkRunnerPreCommit project(path: ":runners:flink:1.5", configuration: "shadow") // TODO: Make the netty version used configurable, we add netty-all 4.1.17.Final so it appears on the classpath // before 4.1.8.Final defined by Apache Beam sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final" - sparkRunnerPreCommit project(path: ":beam-runners-spark", configuration: "shadow") - sparkRunnerPreCommit project(path: ":beam-sdks-java-io-hadoop-file-system", configuration: "shadow") + sparkRunnerPreCommit project(path: ":runners:spark", configuration: "shadow") + sparkRunnerPreCommit project(path: ":sdks:java:io:hadoop-file-system", configuration: "shadow") sparkRunnerPreCommit library.java.spark_streaming sparkRunnerPreCommit library.java.spark_core implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" diff --git a/model/fn-execution/build.gradle b/model/fn-execution/build.gradle index 1d945373e5f05..82f4e81b15b41 100644 --- a/model/fn-execution/build.gradle +++ b/model/fn-execution/build.gradle @@ -25,6 +25,6 @@ ext.summary = "Portable definitions for execution user-defined functions." dependencies { // We purposely depend on the unshaded classes for protobuf compilation and // export the shaded variant as the actual runtime dependency. - compile project(path: ":beam-model-pipeline", configuration: "unshaded") - runtime project(path: ":beam-model-pipeline", configuration: "shadow") + compile project(path: ":model:pipeline", configuration: "unshaded") + runtime project(path: ":model:pipeline", configuration: "shadow") } diff --git a/model/job-management/build.gradle b/model/job-management/build.gradle index 07e9c2aae844d..38c7938a83b9c 100644 --- a/model/job-management/build.gradle +++ b/model/job-management/build.gradle @@ -28,6 +28,6 @@ ext.summary = "Portable definitions for submitting pipelines." dependencies { // We purposely depend on the unshaded classes for protobuf compilation and // export the shaded variant as the actual runtime dependency. - compile project(path: ":beam-model-pipeline", configuration: "unshaded") - runtime project(path: ":beam-model-pipeline", configuration: "shadow") + compile project(path: ":model:pipeline", configuration: "unshaded") + runtime project(path: ":model:pipeline", configuration: "shadow") } diff --git a/release/build.gradle b/release/build.gradle index eabed5bfc074b..c5228babf52e8 100644 --- a/release/build.gradle +++ b/release/build.gradle @@ -30,11 +30,11 @@ dependencies { task runJavaExamplesValidationTask { group = "Verification" description = "Run the Beam quickstart across all Java runners" - dependsOn ":beam-runners-direct-java:runQuickstartJavaDirect" - dependsOn ":beam-runners-google-cloud-dataflow-java:runQuickstartJavaDataflow" - dependsOn ":beam-runners-apex:runQuickstartJavaApex" - dependsOn ":beam-runners-spark:runQuickstartJavaSpark" - dependsOn ":beam-runners-flink_2.11:runQuickstartJavaFlinkLocal" - dependsOn ":beam-runners-direct-java:runMobileGamingJavaDirect" - dependsOn ":beam-runners-google-cloud-dataflow-java:runMobileGamingJavaDataflow" + dependsOn ":runners:direct-java:runQuickstartJavaDirect" + dependsOn ":runners:google-cloud-dataflow-java:runQuickstartJavaDataflow" + dependsOn ":runners:apex:runQuickstartJavaApex" + dependsOn ":runners:spark:runQuickstartJavaSpark" + dependsOn ":runners:flink:1.5:runQuickstartJavaFlinkLocal" + dependsOn ":runners:direct-java:runMobileGamingJavaDirect" + dependsOn ":runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow" } diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh index 8055d6e9cd1bf..0af6bb1f41c8f 100755 --- a/release/src/main/scripts/build_release_candidate.sh +++ b/release/src/main/scripts/build_release_candidate.sh @@ -230,7 +230,7 @@ if [[ $confirmation = "y" ]]; then git clone ${GIT_REPO_URL} cd ${BEAM_ROOT_DIR} git checkout ${RELEASE_BRANCH} - ./gradlew :beam-sdks-java-javadoc:aggregateJavadoc + ./gradlew :sdks:java:javadoc:aggregateJavadoc GENERATE_JAVADOC=~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_JAVA_DOC}/${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/ echo "------------------Updating Release Docs---------------------" diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh index 393e2a0f143b5..3eab8ca8bf6c5 100755 --- a/release/src/main/scripts/run_rc_validation.sh +++ b/release/src/main/scripts/run_rc_validation.sh @@ -113,7 +113,7 @@ if [[ $confirmation = "y" ]]; then echo "*************************************************************" echo "* Running Java Quickstart with DirectRunner" echo "*************************************************************" - ./gradlew :beam-runners-direct-java:runQuickstartJavaDirect \ + ./gradlew :runners:direct-java:runQuickstartJavaDirect \ -Prepourl=${REPO_URL} \ -Pver=${RELEASE} fi @@ -125,7 +125,7 @@ if [[ $confirmation = "y" ]]; then echo "*************************************************************" echo "* Running Java Quickstart with Apex local runner" echo "*************************************************************" - ./gradlew :beam-runners-apex:runQuickstartJavaApex \ + ./gradlew :runners:apex:runQuickstartJavaApex \ -Prepourl=${REPO_URL} \ -Pver=${RELEASE} fi @@ -137,7 +137,7 @@ if [[ $confirmation = "y" ]]; then echo "*************************************************************" echo "* Running Java Quickstart with Flink local runner" echo "*************************************************************" - ./gradlew :beam-runners-flink_2.11:runQuickstartJavaFlinkLocal \ + ./gradlew :runners:flink:1.5:runQuickstartJavaFlinkLocal \ -Prepourl=${REPO_URL} \ -Pver=${RELEASE} fi @@ -149,7 +149,7 @@ if [[ $confirmation = "y" ]]; then echo "*************************************************************" echo "* Running Java Quickstart with Spark local runner" echo "*************************************************************" - ./gradlew :beam-runners-spark:runQuickstartJavaSpark \ + ./gradlew :runners:spark:runQuickstartJavaSpark \ -Prepourl=${REPO_URL} \ -Pver=${RELEASE} fi @@ -184,7 +184,7 @@ if [[ $confirmation = "y" ]]; then echo "*************************************************************" gcloud auth application-default login gcloud config set project ${USER_GCP_PROJECT} - ./gradlew :beam-runners-google-cloud-dataflow-java:runQuickstartJavaDataflow \ + ./gradlew :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow \ -Prepourl=${REPO_URL} \ -Pver=${RELEASE} \ -PgcpProject=${USER_GCP_PROJECT} \ @@ -231,7 +231,7 @@ if [[ $confirmation = "y" ]]; then echo "**************************************************************************" echo "* Java mobile game validations: UserScore, HourlyTeamScore, Leaderboard" echo "**************************************************************************" - ./gradlew :beam-runners-google-cloud-dataflow-java:runMobileGamingJavaDataflow \ + ./gradlew :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow \ -Prepourl=${REPO_URL} \ -Pver=${RELEASE} \ -PgcpProject=${USER_GCP_PROJECT} \ diff --git a/runners/apex/build.gradle b/runners/apex/build.gradle index 8fd3ace649d4d..988847d472f6a 100644 --- a/runners/apex/build.gradle +++ b/runners/apex/build.gradle @@ -28,31 +28,31 @@ description = "Apache Beam :: Runners :: Apex" * the following projects are evaluated before we evaluate this project. This is because * we are attempting to reference the "sourceSets.test.output" directly. */ -evaluationDependsOn(":beam-sdks-java-core") +evaluationDependsOn(":sdks:java:core") configurations { validatesRunner } dependencies { - shadow project(path: ":beam-model-pipeline", configuration: "shadow") - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow") - shadow project(path: ":beam-runners-core-java", configuration: "shadow") + shadow project(path: ":model:pipeline", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":runners:core-construction-java", configuration: "shadow") + shadow project(path: ":runners:core-java", configuration: "shadow") shadow library.java.apex_common shadow library.java.malhar_library shadow library.java.apex_engine shadow library.java.commons_lang3 shadow library.java.apex_engine - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") // ApexStateInternalsTest extends abstract StateInternalsTest - shadowTest project(path: ":beam-runners-core-java", configuration: "shadowTest") + shadowTest project(path: ":runners:core-java", configuration: "shadowTest") shadowTest library.java.hamcrest_core shadowTest library.java.junit shadowTest library.java.mockito_core shadowTest library.java.jackson_dataformat_yaml - validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") - validatesRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") + validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest") + validatesRunner project(path: ":runners:core-java", configuration: "shadowTest") validatesRunner project(path: project.path, configuration: "shadow") } @@ -85,7 +85,7 @@ task validatesRunnerBatch(type: Test) { ]) classpath = configurations.validatesRunner - testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders' @@ -110,5 +110,5 @@ task validatesRunner { dependsOn validatesRunnerBatch } -// Generates :beam-runners-apex:runQuickstartJavaApex +// Generates :runners:apex:runQuickstartJavaApex createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner:'Apex') diff --git a/runners/core-construction-java/build.gradle b/runners/core-construction-java/build.gradle index d1b182533a926..b24927d72a623 100644 --- a/runners/core-construction-java/build.gradle +++ b/runners/core-construction-java/build.gradle @@ -33,9 +33,9 @@ test { } dependencies { - shadow project(path: ":beam-model-pipeline", configuration: "shadow") - shadow project(path: ":beam-model-job-management", configuration: "shadow") - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":model:pipeline", configuration: "shadow") + shadow project(path: ":model:job-management", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.vendored_grpc_1_13_1 shadow library.java.vendored_guava_20_0 shadow library.java.jackson_core @@ -48,7 +48,7 @@ dependencies { shadowTest library.java.mockito_core shadowTest library.java.jackson_annotations shadowTest library.java.jackson_dataformat_yaml - shadowTest project(path: ":beam-model-fn-execution", configuration: "shadow") + shadowTest project(path: ":model:fn-execution", configuration: "shadow") } task runExpansionService (type: JavaExec) { diff --git a/runners/core-java/build.gradle b/runners/core-java/build.gradle index 12f3936246a5d..f5ab4aee0baa4 100644 --- a/runners/core-java/build.gradle +++ b/runners/core-java/build.gradle @@ -31,14 +31,14 @@ test { } dependencies { - shadow project(path: ":beam-model-pipeline", configuration: "shadow") - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-model-fn-execution", configuration: "shadow") - shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow") - shadow project(path: ":beam-sdks-java-fn-execution", configuration: "shadow") + shadow project(path: ":model:pipeline", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":model:fn-execution", configuration: "shadow") + shadow project(path: ":runners:core-construction-java", configuration: "shadow") + shadow project(path: ":sdks:java:fn-execution", configuration: "shadow") shadow library.java.vendored_guava_20_0 shadow library.java.joda_time - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") shadowTest library.java.junit shadowTest library.java.hamcrest_core shadowTest library.java.hamcrest_library diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoMetricName.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoMetricName.java index 9143eff0055fc..22f761906b0be 100644 --- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoMetricName.java +++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MonitoringInfoMetricName.java @@ -40,7 +40,7 @@ public class MonitoringInfoMetricName extends MetricName { private MonitoringInfoMetricName(String urn, Map labels) { checkArgument(!Strings.isNullOrEmpty(urn), "MonitoringInfoMetricName urn must be non-empty"); checkArgument(labels != null, "MonitoringInfoMetricName labels must be non-null"); - // TODO(ajamato): Move SimpleMonitoringInfoBuilder to beam-runner-core-construction-java + // TODO(ajamato): Move SimpleMonitoringInfoBuilder to :runners:core-construction-java // and ensure all necessary labels are set for the specific URN. this.urn = urn; for (Entry entry : labels.entrySet()) { diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle index 3b905650eed16..328de712f5a38 100644 --- a/runners/direct-java/build.gradle +++ b/runners/direct-java/build.gradle @@ -22,11 +22,11 @@ plugins { id 'org.apache.beam.module' } // Shade away runner execution utilities till because this causes ServiceLoader conflicts with // TransformPayloadTranslatorRegistrar amongst other runners. This only happens in the DirectRunner // because it is likely to appear on the classpath of another runner. -def dependOnProjects = [":beam-runners-core-construction-java", - ":beam-runners-core-java", - ":beam-runners-local-java-core", - ":beam-runners-java-fn-execution", - ":beam-sdks-java-fn-execution"] +def dependOnProjects = [":runners:core-construction-java", + ":runners:core-java", + ":runners:local-java", + ":runners:java-fn-execution", + ":sdks:java:fn-execution"] applyJavaNature(shadowClosure: DEFAULT_SHADOW_CLOSURE << { dependencies { @@ -47,9 +47,9 @@ description = "Apache Beam :: Runners :: Direct Java" * the following projects are evaluated before we evaluate this project. This is because * we are attempting to reference the "sourceSets.test.output" directly. */ -evaluationDependsOn(":beam-runners-core-construction-java") -evaluationDependsOn(":beam-runners-core-java") -evaluationDependsOn(":beam-sdks-java-core") +evaluationDependsOn(":runners:core-construction-java") +evaluationDependsOn(":runners:core-java") +evaluationDependsOn(":sdks:java:core") configurations { needsRunner @@ -59,20 +59,20 @@ configurations { dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-model-pipeline", configuration: "shadow") + shadow project(path: ":model:pipeline", configuration: "shadow") dependOnProjects.each { compile project(path: it, configuration: "shadow") } - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.vendored_grpc_1_13_1 shadow library.java.joda_time shadow library.java.slf4j_api shadow library.java.args4j provided library.java.hamcrest_core provided library.java.junit - testRuntime project(path: ":beam-sdks-java-harness") - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") - shadowTest project(path: ":beam-runners-core-java", configuration: "shadowTest") + testRuntime project(path: ":sdks:java:harness") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") + shadowTest project(path: ":runners:core-java", configuration: "shadowTest") shadowTest library.java.guava_testlib shadowTest library.java.slf4j_jdk14 shadowTest library.java.mockito_core @@ -80,21 +80,21 @@ dependencies { shadowTest library.java.woodstox_core_asl shadowTest library.java.google_cloud_dataflow_java_proto_library_all shadowTest library.java.jackson_dataformat_yaml - needsRunner project(path: ":beam-runners-core-construction-java", configuration: "shadowTest") - needsRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") - needsRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") + needsRunner project(path: ":runners:core-construction-java", configuration: "shadowTest") + needsRunner project(path: ":runners:core-java", configuration: "shadowTest") + needsRunner project(path: ":sdks:java:core", configuration: "shadowTest") needsRunner project(path: project.path, configuration: "shadow") needsRunner project(path: project.path, configuration: "shadowTest") - validatesRunner project(path: ":beam-runners-core-construction-java", configuration: "shadowTest") - validatesRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") - validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") + validatesRunner project(path: ":runners:core-construction-java", configuration: "shadowTest") + validatesRunner project(path: ":runners:core-java", configuration: "shadowTest") + validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest") validatesRunner project(path: project.path, configuration: "shadow") validatesRunner project(path: project.path, configuration: "shadowTest") - validatesPortableRunner project(path: ":beam-runners-core-construction-java", configuration: "shadowTest") - validatesPortableRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") - validatesPortableRunner project(path: ":beam-runners-java-fn-execution", configuration: "shadowTest") - validatesPortableRunner project(path: ":beam-runners-reference-java", configuration: "shadowTest") - validatesPortableRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") + validatesPortableRunner project(path: ":runners:core-construction-java", configuration: "shadowTest") + validatesPortableRunner project(path: ":runners:core-java", configuration: "shadowTest") + validatesPortableRunner project(path: ":runners:java-fn-execution", configuration: "shadowTest") + validatesPortableRunner project(path: ":runners:reference:java", configuration: "shadowTest") + validatesPortableRunner project(path: ":sdks:java:core", configuration: "shadowTest") validatesPortableRunner project(path: project.path, configuration: "shadow") validatesPortableRunner project(path: project.path, configuration: "shadowTest") } @@ -109,9 +109,9 @@ task needsRunnerTests(type: Test) { classpath = configurations.needsRunner // NOTE: We only add to the test class dirs to ensure that the direct runner // tests (i.e., those from this subproject) get scanned. - testClassesDirs += files(project(":beam-runners-core-construction-java").sourceSets.test.output.classesDirs) - testClassesDirs += files(project(":beam-runners-core-java").sourceSets.test.output.classesDirs) - testClassesDirs += files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs += files(project(":runners:core-construction-java").sourceSets.test.output.classesDirs) + testClassesDirs += files(project(":runners:core-java").sourceSets.test.output.classesDirs) + testClassesDirs += files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { includeCategories "org.apache.beam.sdk.testing.NeedsRunner" excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB" @@ -134,9 +134,9 @@ task validatesRunner(type: Test) { classpath = configurations.validatesRunner // NOTE: We only add to the test class dirs to ensure that the direct runner // tests (i.e., those from this subproject) get scanned. - testClassesDirs += files(project(":beam-runners-core-construction-java").sourceSets.test.output.classesDirs) - testClassesDirs += files(project(":beam-runners-core-java").sourceSets.test.output.classesDirs) - testClassesDirs += files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs += files(project(":runners:core-construction-java").sourceSets.test.output.classesDirs) + testClassesDirs += files(project(":runners:core-java").sourceSets.test.output.classesDirs) + testClassesDirs += files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { includeCategories "org.apache.beam.sdk.testing.ValidatesRunner" excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB" @@ -150,10 +150,10 @@ def gcsBucket = project.findProperty('gcsBucket') ?: 'temp-storage-for-release-v def bqDataset = project.findProperty('bqDataset') ?: 'beam_postrelease_mobile_gaming' def pubsubTopic = project.findProperty('pubsubTopic') ?: 'java_mobile_gaming_topic' -// Generates :beam-runners-direct-java:runQuickstartJavaDirect +// Generates :runners:direct-java:runQuickstartJavaDirect createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'Direct') -// Generates :beam-runners-direct-java:runMobileGamingJavaDirect +// Generates :runners:direct-java:runMobileGamingJavaDirect createJavaExamplesArchetypeValidationTask(type: 'MobileGaming', runner: 'Direct', gcpProject: gcpProject, diff --git a/runners/extensions-java/metrics/build.gradle b/runners/extensions-java/metrics/build.gradle index a987b82bc45bb..455e5be819a4f 100644 --- a/runners/extensions-java/metrics/build.gradle +++ b/runners/extensions-java/metrics/build.gradle @@ -24,7 +24,7 @@ ext.summary = "Beam Runners Extensions Metrics provides implementations of runne dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.jackson_databind shadow library.java.jackson_datatype_joda shadowTest library.java.joda_time diff --git a/runners/flink/1.5/build.gradle b/runners/flink/1.5/build.gradle index 09545574b0c86..b06339534ddc0 100644 --- a/runners/flink/1.5/build.gradle +++ b/runners/flink/1.5/build.gradle @@ -27,6 +27,7 @@ project.ext { test_source_dirs = ["$basePath/src/test/java", "./src/test/java"] main_resources_dirs = ["$basePath/src/main/resources"] test_resources_dirs = ["$basePath/src/test/resources"] + archives_base_name = 'beam-runners-flink_2.11' } // Load the main build script which contains all build logic. diff --git a/runners/flink/1.5/job-server/build.gradle b/runners/flink/1.5/job-server/build.gradle index d5251e048213b..fbba7a34ac545 100644 --- a/runners/flink/1.5/job-server/build.gradle +++ b/runners/flink/1.5/job-server/build.gradle @@ -24,6 +24,7 @@ project.ext { test_source_dirs = ["$basePath/src/test/java"] main_resources_dirs = ["$basePath/src/main/resources"] test_resources_dirs = ["$basePath/src/test/resources"] + archives_base_name = 'beam-runners-flink_2.11-job-server' } // Load the main build script which contains all build logic. diff --git a/runners/flink/1.6/build.gradle b/runners/flink/1.6/build.gradle index 39f162814e333..e8541cc4a4cd3 100644 --- a/runners/flink/1.6/build.gradle +++ b/runners/flink/1.6/build.gradle @@ -27,6 +27,7 @@ project.ext { test_source_dirs = ["$basePath/src/test/java", "../1.5/src/test/java"] main_resources_dirs = ["$basePath/src/main/resources"] test_resources_dirs = ["$basePath/src/test/resources"] + archives_base_name = 'beam-runners-flink-1.6' } // Load the main build script which contains all build logic. diff --git a/runners/flink/1.6/job-server/build.gradle b/runners/flink/1.6/job-server/build.gradle index d5251e048213b..39f1810da65b0 100644 --- a/runners/flink/1.6/job-server/build.gradle +++ b/runners/flink/1.6/job-server/build.gradle @@ -24,6 +24,7 @@ project.ext { test_source_dirs = ["$basePath/src/test/java"] main_resources_dirs = ["$basePath/src/main/resources"] test_resources_dirs = ["$basePath/src/test/resources"] + archives_base_name = 'beam-runners-flink-1.6-job-server' } // Load the main build script which contains all build logic. diff --git a/runners/flink/1.7/build.gradle b/runners/flink/1.7/build.gradle index 5bb6e0e1e92a0..9029153aef44c 100644 --- a/runners/flink/1.7/build.gradle +++ b/runners/flink/1.7/build.gradle @@ -27,6 +27,7 @@ project.ext { test_source_dirs = ["$basePath/src/test/java", "../1.5/src/test/java"] main_resources_dirs = ["$basePath/src/main/resources"] test_resources_dirs = ["$basePath/src/test/resources"] + archives_base_name = 'beam-runners-flink-1.7' } // Load the main build script which contains all build logic. diff --git a/runners/flink/1.7/job-server/build.gradle b/runners/flink/1.7/job-server/build.gradle index d5251e048213b..7b4d15e17f2a0 100644 --- a/runners/flink/1.7/job-server/build.gradle +++ b/runners/flink/1.7/job-server/build.gradle @@ -24,6 +24,7 @@ project.ext { test_source_dirs = ["$basePath/src/test/java"] main_resources_dirs = ["$basePath/src/main/resources"] test_resources_dirs = ["$basePath/src/test/resources"] + archives_base_name = 'beam-runners-flink-1.7-job-server' } // Load the main build script which contains all build logic. diff --git a/runners/flink/1.8/build.gradle b/runners/flink/1.8/build.gradle index 9dd377f550c27..41ca8fc0c46c1 100644 --- a/runners/flink/1.8/build.gradle +++ b/runners/flink/1.8/build.gradle @@ -27,6 +27,7 @@ project.ext { test_source_dirs = ["$basePath/src/test/java", "./src/test/java"] main_resources_dirs = ["$basePath/src/main/resources"] test_resources_dirs = ["$basePath/src/test/resources"] + archives_base_name = 'beam-runners-flink-1.8' } // Load the main build script which contains all build logic. diff --git a/runners/flink/1.8/job-server/build.gradle b/runners/flink/1.8/job-server/build.gradle index d5251e048213b..562b6ca6480d1 100644 --- a/runners/flink/1.8/job-server/build.gradle +++ b/runners/flink/1.8/job-server/build.gradle @@ -24,6 +24,7 @@ project.ext { test_source_dirs = ["$basePath/src/test/java"] main_resources_dirs = ["$basePath/src/main/resources"] test_resources_dirs = ["$basePath/src/test/resources"] + archives_base_name = 'beam-runners-flink-1.8-job-server' } // Load the main build script which contains all build logic. diff --git a/runners/flink/flink_runner.gradle b/runners/flink/flink_runner.gradle index 94cd3a23c618f..39fa777d8bf16 100644 --- a/runners/flink/flink_runner.gradle +++ b/runners/flink/flink_runner.gradle @@ -26,6 +26,7 @@ import groovy.json.JsonOutput apply plugin: 'org.apache.beam.module' +archivesBaseName = project.hasProperty('archives_base_name') ? archives_base_name : archivesBaseName applyJavaNature() description = "Apache Beam :: Runners :: Flink $flink_version" @@ -35,8 +36,8 @@ description = "Apache Beam :: Runners :: Flink $flink_version" * the following projects are evaluated before we evaluate this project. This is because * we are attempting to reference the "sourceSets.test.output" directly. */ -evaluationDependsOn(":beam-sdks-java-core") -evaluationDependsOn(":beam-runners-core-java") +evaluationDependsOn(":sdks:java:core") +evaluationDependsOn(":runners:core-java") /* * We have to explicitly set all directories here to make sure each @@ -70,15 +71,15 @@ test { } // TODO Running tests of all Flink versions in parallel can be too harsh on Jenkins memory // Run them serially for now, to avoid "Exit code 137", i.e. Jenkins host killing the Gradle test process - if (project.name == "beam-runners-flink-1.6") { - mustRunAfter(":beam-runners-flink_2.11:test") - } else if (project.name == "beam-runners-flink-1.7") { - mustRunAfter(":beam-runners-flink_2.11:test") - mustRunAfter(":beam-runners-flink-1.6:test") - } else if (project.name == "beam-runners-flink-1.8") { - mustRunAfter(":beam-runners-flink_2.11:test") - mustRunAfter(":beam-runners-flink-1.6:test") - mustRunAfter(":beam-runners-flink-1.7:test") + if (project.path == ":runners:flink:1.6") { + mustRunAfter(":runners:flink:1.5:test") + } else if (project.path == ":runners:flink:1.7") { + mustRunAfter(":runners:flink:1.5:test") + mustRunAfter(":runners:flink:1.6:test") + } else if (project.path == ":runners:flink:1.8") { + mustRunAfter(":runners:flink:1.5:test") + mustRunAfter(":runners:flink:1.6:test") + mustRunAfter(":runners:flink:1.7:test") } } @@ -88,11 +89,11 @@ configurations { dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-runners-core-java", configuration: "shadow") - shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow") - shadow project(path: ":beam-runners-java-fn-execution", configuration: "shadow") - shadow project(path: ":beam-sdks-java-build-tools", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":runners:core-java", configuration: "shadow") + shadow project(path: ":runners:core-construction-java", configuration: "shadow") + shadow project(path: ":runners:java-fn-execution", configuration: "shadow") + shadow project(path: ":sdks:java:build-tools", configuration: "shadow") shadow library.java.vendored_grpc_1_13_1 shadow library.java.jackson_annotations shadow library.java.slf4j_api @@ -105,24 +106,24 @@ dependencies { shadow "org.apache.flink:flink-java:$flink_version" shadow "org.apache.flink:flink-runtime_2.11:$flink_version" shadow "org.apache.flink:flink-streaming-java_2.11:$flink_version" - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") // FlinkStateInternalsTest extends abstract StateInternalsTest - shadowTest project(path: ":beam-runners-core-java", configuration: "shadowTest") + shadowTest project(path: ":runners:core-java", configuration: "shadowTest") shadowTest library.java.commons_lang3 shadowTest library.java.hamcrest_core shadowTest library.java.junit shadowTest library.java.mockito_core shadowTest library.java.google_api_services_bigquery - shadowTest project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") + shadowTest project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow") shadowTest library.java.jackson_dataformat_yaml shadowTest "org.apache.flink:flink-core:$flink_version:tests" shadowTest "org.apache.flink:flink-runtime_2.11:$flink_version:tests" shadowTest "org.apache.flink:flink-streaming-java_2.11:$flink_version:tests" shadowTest "org.apache.flink:flink-test-utils_2.11:$flink_version" - shadowTest project(":beam-sdks-java-harness") + shadowTest project(":sdks:java:harness") testRuntimeOnly library.java.slf4j_simple - validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") - validatesRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") + validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest") + validatesRunner project(path: ":runners:core-java", configuration: "shadowTest") validatesRunner project(path: project.path, configuration: "shadow") } @@ -144,7 +145,7 @@ def createValidatesRunnerTask(Map m) { ]) systemProperty "beamTestPipelineOptions", pipelineOptions classpath = configurations.validatesRunner - testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs, project(":beam-runners-core-java").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs, project(":runners:core-java").sourceSets.test.output.classesDirs) // maxParallelForks decreased from 4 in order to avoid OOM errors maxParallelForks 2 useJUnit { @@ -173,5 +174,5 @@ task validatesRunner { dependsOn validatesRunnerStreaming } -// Generates :beam-runners-flink_2.11:runQuickstartJavaFlinkLocal +// Generates :runners:flink:1.5:runQuickstartJavaFlinkLocal createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'FlinkLocal') diff --git a/runners/flink/job-server-container/flink_job_server_container.gradle b/runners/flink/job-server-container/flink_job_server_container.gradle index dd51777b51e8d..acaa3c3daffae 100644 --- a/runners/flink/job-server-container/flink_job_server_container.gradle +++ b/runners/flink/job-server-container/flink_job_server_container.gradle @@ -27,7 +27,7 @@ apply plugin: 'base' applyDockerNature() // Resolve the Flink project name (and version) the job-server-container is based on -def flinkJobServerProject = ":${project.name.replace('-container', '')}" +def flinkJobServerProject = "${project.path.replace('-container', '')}" description = project(flinkJobServerProject).description + " :: Container" diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle index 9a8866afd0275..e9f19ca1f6798 100644 --- a/runners/flink/job-server/flink_job_server.gradle +++ b/runners/flink/job-server/flink_job_server.gradle @@ -28,6 +28,7 @@ apply plugin: 'org.apache.beam.module' apply plugin: 'application' // we need to set mainClassName before applying shadow plugin mainClassName = "org.apache.beam.runners.flink.FlinkJobServerDriver" +archivesBaseName = project.hasProperty('archives_base_name') ? archives_base_name : archivesBaseName applyJavaNature( validateShadowJar: false, @@ -38,7 +39,7 @@ applyJavaNature( ) // Resolve the Flink project name (and version) the job-server is based on -def flinkRunnerProject = ":${project.name.replace("-job-server", "")}" +def flinkRunnerProject = "${project.path.replace(":job-server", "")}" description = project(flinkRunnerProject).description + " :: Job Server" @@ -78,14 +79,14 @@ dependencies { compile project(path: flinkRunnerProject, configuration: "shadow") compile group: "org.slf4j", name: "jcl-over-slf4j", version: dependencies.create(project.library.java.slf4j_api).getVersion() validatesPortableRunner project(path: flinkRunnerProject, configuration: "shadowTest") - validatesPortableRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") - validatesPortableRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") - validatesPortableRunner project(path: ":beam-runners-reference-java", configuration: "shadowTest") - compile project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadow") + validatesPortableRunner project(path: ":sdks:java:core", configuration: "shadowTest") + validatesPortableRunner project(path: ":runners:core-java", configuration: "shadowTest") + validatesPortableRunner project(path: ":runners:reference:java", configuration: "shadowTest") + compile project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadow") compile library.java.slf4j_simple // TODO: Enable AWS and HDFS file system. // For resolving external transform requests - compile project(path: ":beam-sdks-java-io-kafka", configuration: "shadow") + compile project(path: ":sdks:java:io:kafka", configuration: "shadow") } // NOTE: runShadow must be used in order to run the job server. The standard run @@ -180,6 +181,6 @@ project.ext.validatesCrossLanguageTransforms = includeCategories 'org.apache.beam.sdk.testing.UsesCrossLanguageTransforms' }, ) -project.evaluationDependsOn ':beam-sdks-python' -validatesCrossLanguageTransforms.dependsOn ':beam-sdks-python:setupVirtualenv' -validatesCrossLanguageTransforms.systemProperty "pythonTestExpansionCommand", ". ${project(':beam-sdks-python').envdir}/bin/activate && pip install -e ${project(':beam-sdks-python').projectDir}[test] && python -m apache_beam.runners.portability.expansion_service_test" +project.evaluationDependsOn ':sdks:python' +validatesCrossLanguageTransforms.dependsOn ':sdks:python:setupVirtualenv' +validatesCrossLanguageTransforms.systemProperty "pythonTestExpansionCommand", ". ${project(':sdks:python').envdir}/bin/activate && pip install -e ${project(':sdks:python').projectDir}[test] && python -m apache_beam.runners.portability.expansion_service_test" diff --git a/runners/gearpump/build.gradle b/runners/gearpump/build.gradle index b95f987e99877..8bae691303d13 100644 --- a/runners/gearpump/build.gradle +++ b/runners/gearpump/build.gradle @@ -28,7 +28,7 @@ description = "Apache Beam :: Runners :: Gearpump" * the following projects are evaluated before we evaluate this project. This is because * we are attempting to reference the "sourceSets.test.output" directly. */ -evaluationDependsOn(":beam-sdks-java-core") +evaluationDependsOn(":sdks:java:core") def gearpump_version = "0.9.0" @@ -40,22 +40,22 @@ dependencies { shadow library.java.vendored_guava_20_0 compileOnly "com.typesafe:config:1.3.0" compileOnly "org.scala-lang:scala-library:2.12.7" - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-runners-core-java", configuration: "shadow") - shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":runners:core-java", configuration: "shadow") + shadow project(path: ":runners:core-construction-java", configuration: "shadow") shadow "io.github.gearpump:gearpump-core_2.12:$gearpump_version:assembly" shadow "io.github.gearpump:gearpump-streaming_2.12:$gearpump_version:assembly" shadow library.java.joda_time shadow library.java.jackson_annotations - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") shadowTest library.java.junit shadowTest library.java.hamcrest_core shadowTest library.java.hamcrest_library shadowTest library.java.jackson_databind shadowTest library.java.jackson_dataformat_yaml shadowTest library.java.mockito_core - validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") - validatesRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") + validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest") + validatesRunner project(path: ":runners:core-java", configuration: "shadowTest") validatesRunner project(path: project.path, configuration: "shadow") } @@ -68,7 +68,7 @@ task validatesRunnerStreaming(type: Test) { classpath = configurations.validatesRunner - testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders' diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 1bce3241521fc..47d91fa6bc415 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -28,12 +28,12 @@ description = "Apache Beam :: Runners :: Google Cloud Dataflow" * the following projects are evaluated before we evaluate this project. This is because * we are attempting to reference parameters such as "sourceSets.test.output" directly. */ -evaluationDependsOn(":beam-sdks-java-io-google-cloud-platform") -evaluationDependsOn(":beam-sdks-java-core") -evaluationDependsOn(":beam-examples-java") -evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker") -evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-fn-api-worker") -evaluationDependsOn(":beam-sdks-java-container") +evaluationDependsOn(":sdks:java:io:google-cloud-platform") +evaluationDependsOn(":sdks:java:core") +evaluationDependsOn(":examples:java") +evaluationDependsOn(":runners:google-cloud-dataflow-java:worker:legacy-worker") +evaluationDependsOn(":runners:google-cloud-dataflow-java:worker") +evaluationDependsOn(":sdks:java:container") processResources { filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ @@ -58,11 +58,11 @@ configurations { dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-model-pipeline", configuration: "shadow") - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") - shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow") + shadow project(path: ":model:pipeline", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadow") + shadow project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow") + shadow project(path: ":runners:core-construction-java", configuration: "shadow") shadow library.java.vendored_grpc_1_13_1 shadow library.java.google_api_client shadow library.java.google_http_client @@ -81,25 +81,25 @@ dependencies { shadow library.java.slf4j_api shadowTest library.java.hamcrest_core shadowTest library.java.junit - shadowTest project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadowTest") shadowTest library.java.guava_testlib shadowTest library.java.slf4j_jdk14 shadowTest library.java.mockito_core shadowTest library.java.google_cloud_dataflow_java_proto_library_all shadowTest library.java.datastore_v1_protos shadowTest library.java.jackson_dataformat_yaml - validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") + validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest") validatesRunner project(path: project.path, configuration: "shadow") validatesRunner library.java.hamcrest_core validatesRunner library.java.hamcrest_library coreSDKJavaIntegrationTest project(path: project.path, configuration: "shadow") - coreSDKJavaIntegrationTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") + coreSDKJavaIntegrationTest project(path: ":sdks:java:core", configuration: "shadowTest") examplesJavaIntegrationTest project(path: project.path, configuration: "shadow") - examplesJavaIntegrationTest project(path: ":beam-examples-java", configuration: "shadowTest") + examplesJavaIntegrationTest project(path: ":examples:java", configuration: "shadowTest") googleCloudPlatformIntegrationTest project(path: project.path, configuration: "shadow") - googleCloudPlatformIntegrationTest project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadowTest") + googleCloudPlatformIntegrationTest project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadowTest") } test { @@ -111,8 +111,8 @@ def dataflowValidatesTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs: def dataflowPostCommitTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests' def dataflowUploadTemp = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-upload-tests' def testFilesToStage = project.findProperty('filesToStage') ?: 'test.txt' -def dataflowLegacyWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath -def dataflowFnApiWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath +def dataflowLegacyWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath +def dataflowFnApiWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath def dataflowKmsKey = project.findProperty('dataflowKmsKey') ?: "projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test" def dockerImageRoot = project.findProperty('dockerImageRoot') ?: "us.gcr.io/${dataflowProject}/java-postcommit-it" @@ -156,7 +156,7 @@ def fnApiWorkerExcludeCategories = [ // make Dataflow pick up the non-versioned container image, which handles a staged worker jar. task validatesRunnerLegacyWorkerTest(type: Test) { group = "Verification" - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ "--runner=TestDataflowRunner", @@ -170,7 +170,7 @@ task validatesRunnerLegacyWorkerTest(type: Test) { // to the number of CPU cores, but can be increased by setting --max-workers=N. maxParallelForks Integer.MAX_VALUE classpath = configurations.validatesRunner - testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' commonExcludeCategories.each { @@ -181,7 +181,7 @@ task validatesRunnerLegacyWorkerTest(type: Test) { task validatesRunnerLegacyWorkerJava11Test(type: Test) { group = "Verification" - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ "--runner=TestDataflowRunner", "--project=${dataflowProject}", @@ -195,7 +195,7 @@ task validatesRunnerLegacyWorkerJava11Test(type: Test) { // to the number of CPU cores, but can be increased by setting --max-workers=N. maxParallelForks Integer.MAX_VALUE classpath = configurations.validatesRunner - testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' commonExcludeCategories.each { @@ -209,7 +209,7 @@ task validatesRunnerLegacyWorkerJava11Test(type: Test) { // task directly ('dependsOn buildAndPushDockerContainer'). This ensures the correct // task ordering such that the registry doesn't get cleaned up prior to task completion. task buildAndPushDockerContainer() { - dependsOn ":beam-sdks-java-container:docker" + dependsOn ":sdks:java:container:docker" finalizedBy 'cleanUpDockerImages' def defaultDockerImageName = containerImageName(name: "java") doLast { @@ -235,7 +235,7 @@ task printFnApiPipelineOptions { group = "Help" description = "Prints to the console extra pipeline options needed to run a Dataflow pipeline using portability" - dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" dependsOn buildAndPushDockerContainer doLast { @@ -246,7 +246,7 @@ task printFnApiPipelineOptions { task validatesRunnerFnApiWorkerTest(type: Test) { group = "Verification" - dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" dependsOn buildAndPushDockerContainer systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ "--runner=TestDataflowRunner", @@ -258,7 +258,7 @@ task validatesRunnerFnApiWorkerTest(type: Test) { // to the number of CPU cores, but can be increased by setting --max-workers=N. maxParallelForks Integer.MAX_VALUE classpath = configurations.validatesRunner - testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' commonExcludeCategories.each { @@ -273,7 +273,7 @@ task validatesRunnerFnApiWorkerTest(type: Test) { task validatesRunnerFnApiWorkerExecutableStageTest(type: Test) { group = "Verification" description "Validates Dataflow PortabilityApi runner" - dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" dependsOn buildAndPushDockerContainer systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ @@ -289,7 +289,7 @@ task validatesRunnerFnApiWorkerExecutableStageTest(type: Test) { // to the number of CPU cores, but can be increased by setting --max-workers=N. maxParallelForks Integer.MAX_VALUE classpath = configurations.validatesRunner - testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' commonExcludeCategories.each { @@ -317,7 +317,7 @@ task validatesJava11Runner { task googleCloudPlatformLegacyWorkerIntegrationTest(type: Test) { group = "Verification" - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ "--runner=TestDataflowRunner", "--project=${dataflowProject}", @@ -333,13 +333,13 @@ task googleCloudPlatformLegacyWorkerIntegrationTest(type: Test) { exclude '**/*KmsKeyIT.class' maxParallelForks 4 classpath = configurations.googleCloudPlatformIntegrationTest - testClassesDirs = files(project(":beam-sdks-java-io-google-cloud-platform").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:io:google-cloud-platform").sourceSets.test.output.classesDirs) useJUnit { } } task googleCloudPlatformLegacyWorkerKmsIntegrationTest(type: Test) { group = "Verification" - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ "--runner=TestDataflowRunner", "--project=${dataflowProject}", @@ -353,13 +353,13 @@ task googleCloudPlatformLegacyWorkerKmsIntegrationTest(type: Test) { exclude '**/BigQueryKmsKeyIT.class' // Only needs to run on direct runner. maxParallelForks 4 classpath = configurations.googleCloudPlatformIntegrationTest - testClassesDirs = files(project(":beam-sdks-java-io-google-cloud-platform").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:io:google-cloud-platform").sourceSets.test.output.classesDirs) useJUnit { } } task googleCloudPlatformFnApiWorkerIntegrationTest(type: Test) { group = "Verification" - dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" dependsOn buildAndPushDockerContainer systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ @@ -384,7 +384,7 @@ task googleCloudPlatformFnApiWorkerIntegrationTest(type: Test) { maxParallelForks 4 classpath = configurations.googleCloudPlatformIntegrationTest - testClassesDirs = files(project(":beam-sdks-java-io-google-cloud-platform").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:io:google-cloud-platform").sourceSets.test.output.classesDirs) useJUnit { excludeCategories 'org.apache.beam.sdk.testing.DataflowPortabilityApiUnsupported' } @@ -392,7 +392,7 @@ task googleCloudPlatformFnApiWorkerIntegrationTest(type: Test) { task examplesJavaLegacyWorkerIntegrationTest(type: Test) { group = "Verification" - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ "--runner=TestDataflowRunner", @@ -409,7 +409,7 @@ task examplesJavaLegacyWorkerIntegrationTest(type: Test) { exclude '**/WindowedWordCountIT.class' maxParallelForks 4 classpath = configurations.examplesJavaIntegrationTest - testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs) useJUnit { } } @@ -417,7 +417,7 @@ task examplesJavaLegacyWorkerIntegrationTest(type: Test) { // Should support more ITs in the future. task examplesJavaFnApiWorkerIntegrationTest(type: Test) { group = "Verification" - dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" dependsOn buildAndPushDockerContainer systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ @@ -439,13 +439,13 @@ task examplesJavaFnApiWorkerIntegrationTest(type: Test) { maxParallelForks 4 classpath = configurations.examplesJavaIntegrationTest - testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs) useJUnit { } } task coreSDKJavaLegacyWorkerIntegrationTest(type: Test) { group = "Verification" - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ "--runner=TestDataflowRunner", @@ -460,13 +460,13 @@ task coreSDKJavaLegacyWorkerIntegrationTest(type: Test) { exclude '**/RequiresStableInputIT.class' maxParallelForks 4 classpath = configurations.coreSDKJavaIntegrationTest - testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { } } task coreSDKJavaFnApiWorkerIntegrationTest(type: Test) { group = "Verification" - dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" dependsOn buildAndPushDockerContainer systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ @@ -480,7 +480,7 @@ task coreSDKJavaFnApiWorkerIntegrationTest(type: Test) { exclude '**/RequiresStableInputIT.class' maxParallelForks 4 classpath = configurations.coreSDKJavaIntegrationTest - testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { } } @@ -519,13 +519,13 @@ def gcsBucket = project.findProperty('gcsBucket') ?: 'temp-storage-for-release-v def bqDataset = project.findProperty('bqDataset') ?: 'beam_postrelease_mobile_gaming' def pubsubTopic = project.findProperty('pubsubTopic') ?: 'java_mobile_gaming_topic' -// Generates :beam-runners-google-cloud-dataflow-java:runQuickstartJavaDataflow +// Generates :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'Dataflow', gcpProject: gcpProject, gcsBucket: gcsBucket) -// Generates :beam-runners-google-cloud-dataflow-java:runMobileGamingJavaDataflow +// Generates :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow createJavaExamplesArchetypeValidationTask(type: 'MobileGaming', runner: 'Dataflow', gcpProject: gcpProject, diff --git a/runners/google-cloud-dataflow-java/examples-streaming/build.gradle b/runners/google-cloud-dataflow-java/examples-streaming/build.gradle index c6c91365f9c35..7d67036da5089 100644 --- a/runners/google-cloud-dataflow-java/examples-streaming/build.gradle +++ b/runners/google-cloud-dataflow-java/examples-streaming/build.gradle @@ -22,22 +22,22 @@ plugins { id 'org.apache.beam.module' } applyJavaNature(publish: false, exportJavadoc: false) // Evaluate the given project before this one, to allow referencing // its sourceSets.test.output directly. -evaluationDependsOn(":beam-examples-java") -evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker") +evaluationDependsOn(":examples:java") +evaluationDependsOn(":runners:google-cloud-dataflow-java:worker:legacy-worker") configurations { dataflowStreamingRunnerPreCommit } dependencies { - testRuntimeOnly project(path: ":beam-examples-java", configuration: "shadow") - testRuntimeOnly project(path: ":beam-examples-java", configuration: "shadowTest") - testRuntimeOnly project(path: ":beam-runners-google-cloud-dataflow-java", configuration: "shadow") + testRuntimeOnly project(path: ":examples:java", configuration: "shadow") + testRuntimeOnly project(path: ":examples:java", configuration: "shadowTest") + testRuntimeOnly project(path: ":runners:google-cloud-dataflow-java", configuration: "shadow") } def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' def gcsTempRoot = project.findProperty('gcsTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests/' task windmillPreCommit(type: Test) { - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath // Set workerHarnessContainerImage to empty to make Dataflow pick up the non-versioned container // image, which handles a staged worker jar. @@ -50,7 +50,7 @@ task windmillPreCommit(type: Test) { "--streaming=true", "--enableStreamingEngine", ] - testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs) include "**/WordCountIT.class" forkEvery 1 maxParallelForks 4 @@ -58,8 +58,8 @@ task windmillPreCommit(type: Test) { } task appliancePreCommit(type: Test) { - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath // Set workerHarnessContainerImage to empty to make Dataflow pick up the non-versioned container // image, which handles a staged worker jar. @@ -71,10 +71,10 @@ task appliancePreCommit(type: Test) { "--workerHarnessContainerImage=", "--streaming=true", ] - testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs) include "**/WordCountIT.class" // Don't include WindowedWordCountIT, since it is already included in - // :beam-runners-google-cloud-dataflow-java-examples:preCommit and runs + // :runners:google-cloud-dataflow-java:examples:preCommit and runs // identically (ignores the --streaming flag). forkEvery 1 maxParallelForks 4 diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle b/runners/google-cloud-dataflow-java/examples/build.gradle index e492fef1146b6..f73c75ab9cd29 100644 --- a/runners/google-cloud-dataflow-java/examples/build.gradle +++ b/runners/google-cloud-dataflow-java/examples/build.gradle @@ -22,30 +22,30 @@ plugins { id 'org.apache.beam.module' } applyJavaNature(publish: false, exportJavadoc: false) // Evaluate the given project before this one, to allow referencing // its sourceSets.test.output directly. -evaluationDependsOn(":beam-examples-java") -evaluationDependsOn(":beam-runners-google-cloud-dataflow-java") -evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker") -evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-fn-api-worker") -evaluationDependsOn(":beam-sdks-java-container") +evaluationDependsOn(":examples:java") +evaluationDependsOn(":runners:google-cloud-dataflow-java") +evaluationDependsOn(":runners:google-cloud-dataflow-java:worker:legacy-worker") +evaluationDependsOn(":runners:google-cloud-dataflow-java:worker") +evaluationDependsOn(":sdks:java:container") configurations { dataflowRunnerPreCommit } dependencies { - testRuntimeOnly project(path: ":beam-examples-java", configuration: "shadow") - testRuntimeOnly project(path: ":beam-examples-java", configuration: "shadowTest") - testRuntimeOnly project(path: ":beam-runners-google-cloud-dataflow-java", configuration: "shadow") + testRuntimeOnly project(path: ":examples:java", configuration: "shadow") + testRuntimeOnly project(path: ":examples:java", configuration: "shadowTest") + testRuntimeOnly project(path: ":runners:google-cloud-dataflow-java", configuration: "shadow") } def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' def gcsTempRoot = project.findProperty('gcsTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests/' -def dockerImageName = project(':beam-runners-google-cloud-dataflow-java').ext.dockerImageName +def dockerImageName = project(':runners:google-cloud-dataflow-java').ext.dockerImageName // If -PuseExecutableStage is set, the use_executable_stage_bundle_execution wil be enabled. def fnapiExperiments = project.hasProperty('useExecutableStage') ? 'beam_fn_api,use_executable_stage_bundle_execution' : "beam_fn_api" def commonConfig = { dataflowWorkerJar, workerHarnessContainerImage = '', additionalOptions = [] -> // return the preevaluated configuration closure return { - testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs) include "**/WordCountIT.class" include "**/WindowedWordCountIT.class" forkEvery 1 @@ -63,15 +63,15 @@ def commonConfig = { dataflowWorkerJar, workerHarnessContainerImage = '', additi } task preCommitLegacyWorker(type: Test) { - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath with commonConfig(dataflowWorkerJar) } task verifyFnApiWorker(type: Test) { - dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" - dependsOn ":beam-runners-google-cloud-dataflow-java:buildAndPushDockerContainer" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:buildAndPushDockerContainer" + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath with commonConfig(dataflowWorkerJar, dockerImageName, ["--experiments=${fnapiExperiments}"]) useJUnit { excludeCategories 'org.apache.beam.sdk.testing.StreamingIT' @@ -79,8 +79,8 @@ task verifyFnApiWorker(type: Test) { } task postCommitLegacyWorkerJava11(type: Test) { - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath systemProperty "java.specification.version", "11" with commonConfig(dataflowWorkerJar) } @@ -100,8 +100,8 @@ task verifyPortabilityApi() { afterEvaluate { // Ensure all tasks which use published docker images run before they are cleaned up tasks.each { t -> - if (t.dependsOn.contains(":beam-runners-google-cloud-dataflow-java:buildAndPushDockerContainer")) { - project(':beam-runners-google-cloud-dataflow-java').cleanUpDockerImages.mustRunAfter t + if (t.dependsOn.contains(":runners:google-cloud-dataflow-java:buildAndPushDockerContainer")) { + project(':runners:google-cloud-dataflow-java').cleanUpDockerImages.mustRunAfter t } } } diff --git a/runners/google-cloud-dataflow-java/worker/build.gradle b/runners/google-cloud-dataflow-java/worker/build.gradle index 63e90d711bf23..5ba415aee74f3 100644 --- a/runners/google-cloud-dataflow-java/worker/build.gradle +++ b/runners/google-cloud-dataflow-java/worker/build.gradle @@ -18,6 +18,8 @@ plugins { id 'org.apache.beam.module' } +archivesBaseName = 'beam-runners-google-cloud-dataflow-java-fn-api-worker' + // Set a specific version of 'com.google.apis:google-api-services-dataflow' // by adding -Pdataflow.version= in Gradle command. Otherwise, // 'google_clients_version' defined in BeamModulePlugin will be used as default. @@ -64,17 +66,17 @@ dependencies { // // All main sourceset dependencies here should be listed as compile scope so that the dependencies // are all packaged into a single uber jar allowing the jar to serve as an application. - compile project(path: ":beam-runners-google-cloud-dataflow-java", configuration: "shadow") - compile project(path: ":beam-sdks-java-core", configuration: "shadow") - compile project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadow") - compile project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") - compile project(path: ":beam-model-fn-execution", configuration: "shadow") - compile project(path: ":beam-model-pipeline", configuration: "shadow") - compile project(path: ":beam-runners-core-construction-java", configuration: "shadow") - compile project(path: ":beam-runners-core-java", configuration: "shadow") - compile project(path: ":beam-runners-java-fn-execution", configuration: "shadow") - compile project(path: ":beam-sdks-java-fn-execution", configuration: "shadow") - compile project(path: ":beam-runners-google-cloud-dataflow-java-windmill", configuration: "shadow") + compile project(path: ":runners:google-cloud-dataflow-java", configuration: "shadow") + compile project(path: ":sdks:java:core", configuration: "shadow") + compile project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadow") + compile project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow") + compile project(path: ":model:fn-execution", configuration: "shadow") + compile project(path: ":model:pipeline", configuration: "shadow") + compile project(path: ":runners:core-construction-java", configuration: "shadow") + compile project(path: ":runners:core-java", configuration: "shadow") + compile project(path: ":runners:java-fn-execution", configuration: "shadow") + compile project(path: ":sdks:java:fn-execution", configuration: "shadow") + compile project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow") compile library.java.vendored_grpc_1_13_1 compile google_api_services_dataflow compile library.java.avro @@ -96,11 +98,11 @@ dependencies { // All test sourceset dependencies can be marked as shadowTest since we create an uber jar without // relocating any code. - shadowTest project(path: ":beam-runners-core-java", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-harness", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadowTest") - shadowTest project(path: ":beam-runners-direct-java", configuration: "shadow") + shadowTest project(path: ":runners:core-java", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:harness", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadowTest") + shadowTest project(path: ":runners:direct-java", configuration: "shadow") shadowTest library.java.guava_testlib shadowTest library.java.hamcrest_core shadowTest library.java.hamcrest_library diff --git a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle index ef8a8c507ba25..3594a753e34ed 100644 --- a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle +++ b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle @@ -25,6 +25,8 @@ plugins { id 'org.apache.beam.module' } +archivesBaseName = 'beam-runners-google-cloud-dataflow-java-legacy-worker' + // Set a specific version of 'com.google.apis:google-api-services-dataflow' // by adding -Pdataflow.version= in Gradle command. Otherwise, // 'google_clients_version' defined in BeamModulePlugin will be used as default. @@ -57,11 +59,11 @@ def sdk_provided_dependencies = [ ] def sdk_provided_project_dependencies = [ - ":beam-model-pipeline", - ":beam-runners-google-cloud-dataflow-java", - ":beam-sdks-java-core", - ":beam-sdks-java-extensions-google-cloud-platform-core", - ":beam-sdks-java-io-google-cloud-platform", + ":model:pipeline", + ":runners:google-cloud-dataflow-java", + ":sdks:java:core", + ":sdks:java:extensions:google-cloud-platform-core", + ":sdks:java:io:google-cloud-platform", ] // Exclude unneeded dependencies when building jar @@ -93,7 +95,7 @@ applyJavaNature( shadowClosure: DEFAULT_SHADOW_CLOSURE << { // Each included dependency must also include all of its necessary transitive dependencies // or have them provided by the users pipeline during job submission. Typically a users - // pipeline includes :beam-runners-google-cloud-dataflow-java and its transitive dependencies + // pipeline includes :runners:google-cloud-dataflow-java and its transitive dependencies // so those dependencies don't need to be shaded (bundled and relocated) away. All other // dependencies needed to run the worker must be shaded (bundled and relocated) to prevent // ClassNotFound and/or MethodNotFound errors during pipeline execution. @@ -102,26 +104,26 @@ applyJavaNature( // that the shaded jar is correctly built. dependencies { - include(project(path: ":beam-model-fn-execution", configuration: "shadow")) + include(project(path: ":model:fn-execution", configuration: "shadow")) } relocate("org.apache.beam.model.fnexecution.v1", getWorkerRelocatedPath("org.apache.beam.model.fnexecution.v1")) dependencies { - include(project(path: ":beam-runners-core-construction-java", configuration: "shadow")) - include(project(path: ":beam-runners-core-java", configuration: "shadow")) + include(project(path: ":runners:core-construction-java", configuration: "shadow")) + include(project(path: ":runners:core-java", configuration: "shadow")) } relocate("org.apache.beam.runners.core", getWorkerRelocatedPath("org.apache.beam.runners.core")) relocate("org.apache.beam.repackaged.beam_runners_core_construction_java", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_core_construction_java")) relocate("org.apache.beam.repackaged.beam_runners_core_java", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_core_java")) dependencies { - include(project(path: ":beam-runners-java-fn-execution", configuration: "shadow")) + include(project(path: ":runners:java-fn-execution", configuration: "shadow")) } relocate("org.apache.beam.runners.fnexecution", getWorkerRelocatedPath("org.apache.beam.runners.fnexecution")) relocate("org.apache.beam.repackaged.beam_runners_java_fn_execution", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_java_fn_execution")) dependencies { - include(project(path: ":beam-sdks-java-fn-execution", configuration: "shadow")) + include(project(path: ":sdks:java:fn-execution", configuration: "shadow")) } relocate("org.apache.beam.sdk.fn", getWorkerRelocatedPath("org.apache.beam.sdk.fn")) relocate("org.apache.beam.repackaged.beam_sdks_java_fn_execution", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_sdks_java_fn_execution")) @@ -144,7 +146,7 @@ applyJavaNature( // expect a user pipeline to include it. There is also a JNI component that windmill server relies on which makes // arbitrary relocation more difficult. dependencies { - include(project(path: ":beam-runners-google-cloud-dataflow-java-windmill", configuration: "shadow")) + include(project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow")) } // Include original source files extracted under @@ -168,14 +170,14 @@ configurations { } } -evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-fn-api-worker") +evaluationDependsOn(":runners:google-cloud-dataflow-java:worker") compileJava { - source project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").sourceSets.main.java.srcDirs + source project(":runners:google-cloud-dataflow-java:worker").sourceSets.main.java.srcDirs } compileTestJava { - source project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").sourceSets.test.java.srcDirs + source project(":runners:google-cloud-dataflow-java:worker").sourceSets.test.java.srcDirs } dependencies { @@ -189,12 +191,12 @@ dependencies { provided project(path: it, configuration: "shadow") } - compile project(path: ":beam-model-fn-execution", configuration: "shadow") - compile project(path: ":beam-runners-core-construction-java", configuration: "shadow") - compile project(path: ":beam-runners-core-java", configuration: "shadow") - compile project(path: ":beam-runners-java-fn-execution", configuration: "shadow") - compile project(path: ":beam-sdks-java-fn-execution", configuration: "shadow") - compile project(path: ":beam-runners-google-cloud-dataflow-java-windmill", configuration: "shadow") + compile project(path: ":model:fn-execution", configuration: "shadow") + compile project(path: ":runners:core-construction-java", configuration: "shadow") + compile project(path: ":runners:core-java", configuration: "shadow") + compile project(path: ":runners:java-fn-execution", configuration: "shadow") + compile project(path: ":sdks:java:fn-execution", configuration: "shadow") + compile project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow") shadow library.java.vendored_guava_20_0 compile "org.conscrypt:conscrypt-openjdk:1.1.3:linux-x86_64" compile "org.eclipse.jetty:jetty-server:9.2.10.v20150310" @@ -205,12 +207,12 @@ dependencies { // Any test dependency which intersects with our relocation rules above needs to be relocated // as well and placed within the testCompile configuration. Otherwise we can place it within // the shadowTest configuration. - testCompile project(path: ":beam-runners-core-java", configuration: "shadowTest") + testCompile project(path: ":runners:core-java", configuration: "shadowTest") shadowTest library.java.guava_testlib - shadowTest project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadowTest") - shadowTest project(path: ":beam-runners-direct-java", configuration: "shadow") - shadowTest project(path: ":beam-sdks-java-harness", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadowTest") + shadowTest project(path: ":runners:direct-java", configuration: "shadow") + shadowTest project(path: ":sdks:java:harness", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") shadowTest library.java.hamcrest_core shadowTest library.java.hamcrest_library shadowTest library.java.junit diff --git a/runners/google-cloud-dataflow-java/worker/windmill/build.gradle b/runners/google-cloud-dataflow-java/worker/windmill/build.gradle index 20df1e3d044f5..da0319ea6aa96 100644 --- a/runners/google-cloud-dataflow-java/worker/windmill/build.gradle +++ b/runners/google-cloud-dataflow-java/worker/windmill/build.gradle @@ -17,6 +17,7 @@ */ plugins { id 'org.apache.beam.module' } +archivesBaseName = 'beam-runners-google-cloud-dataflow-java-windmill' applyPortabilityNature(shadowJarValidationExcludes: ["org/apache/beam/runners/dataflow/worker/windmill/**"]) description = "Apache Beam :: Runners :: Google Cloud Dataflow Java :: Windmill" diff --git a/runners/java-fn-execution/build.gradle b/runners/java-fn-execution/build.gradle index f57f4bc5948e0..1d1ccba2fdd1c 100644 --- a/runners/java-fn-execution/build.gradle +++ b/runners/java-fn-execution/build.gradle @@ -22,21 +22,21 @@ description = "Apache Beam :: Runners :: Java Fn Execution" dependencies { shadow library.java.vendored_guava_20_0 - compile project(path: ":beam-runners-core-construction-java", configuration: "shadow") - provided project(path: ":beam-sdks-java-harness") - shadow project(path: ":beam-model-pipeline", configuration: "shadow") - shadow project(path: ":beam-model-fn-execution", configuration: "shadow") - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-fn-execution", configuration: "shadow") - shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow") - shadow project(path: ":beam-vendor-sdks-java-extensions-protobuf", configuration: "shadow") + compile project(path: ":runners:core-construction-java", configuration: "shadow") + provided project(path: ":sdks:java:harness") + shadow project(path: ":model:pipeline", configuration: "shadow") + shadow project(path: ":model:fn-execution", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":sdks:java:fn-execution", configuration: "shadow") + shadow project(path: ":runners:core-construction-java", configuration: "shadow") + shadow project(path: ":vendor:sdks-java-extensions-protobuf", configuration: "shadow") shadow library.java.vendored_grpc_1_13_1 shadow library.java.slf4j_api shadow library.java.args4j - testCompile project(":beam-sdks-java-harness") - testCompile project(path: ":beam-runners-core-construction-java", configuration: "shadow") - testCompile project(path: ":beam-runners-core-java", configuration: "shadowTest") - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") + testCompile project(":sdks:java:harness") + testCompile project(path: ":runners:core-construction-java", configuration: "shadow") + testCompile project(path: ":runners:core-java", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library diff --git a/runners/local-java/build.gradle b/runners/local-java/build.gradle index 7a29370a4383f..fd57a74200f42 100644 --- a/runners/local-java/build.gradle +++ b/runners/local-java/build.gradle @@ -17,6 +17,9 @@ */ plugins { id 'org.apache.beam.module' } + +archivesBaseName = 'beam-runners-local-java-core' + applyJavaNature() description = "Apache Beam :: Runners :: Local Java Core" @@ -28,7 +31,7 @@ dependencies { * but should not be used within this library to execute any UDFs. * TODO: Add an APISurfaceTest to force this to be the case, if possible. */ - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.joda_time shadowTest library.java.hamcrest_core shadowTest library.java.hamcrest_library diff --git a/runners/reference/java/build.gradle b/runners/reference/java/build.gradle index 87a61332fe56d..59d659c5f7d90 100644 --- a/runners/reference/java/build.gradle +++ b/runners/reference/java/build.gradle @@ -31,14 +31,14 @@ configurations { dependencies { shadow library.java.vendored_guava_20_0 compile library.java.hamcrest_library - shadow project(path: ":beam-model-pipeline", configuration: "shadow") - shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow") - shadow project(path: ":beam-runners-java-fn-execution", configuration: "shadow") - shadow project(path: ":beam-sdks-java-fn-execution", configuration: "shadow") - shadow project(path: ":beam-sdks-java-harness", configuration: "shadow") + shadow project(path: ":model:pipeline", configuration: "shadow") + shadow project(path: ":runners:core-construction-java", configuration: "shadow") + shadow project(path: ":runners:java-fn-execution", configuration: "shadow") + shadow project(path: ":sdks:java:fn-execution", configuration: "shadow") + shadow project(path: ":sdks:java:harness", configuration: "shadow") shadow library.java.vendored_grpc_1_13_1 shadow library.java.slf4j_api - shadowTest project(path: ":beam-runners-core-construction-java", configuration: "shadowTest") + shadowTest project(path: ":runners:core-construction-java", configuration: "shadowTest") shadowTest library.java.guava shadowTest library.java.hamcrest_core shadowTest library.java.junit diff --git a/runners/reference/job-server/build.gradle b/runners/reference/job-server/build.gradle index 2a2f79ddb30c6..53459a89d12ed 100644 --- a/runners/reference/job-server/build.gradle +++ b/runners/reference/job-server/build.gradle @@ -32,8 +32,8 @@ applyJavaNature( description = "Apache Beam :: Runners :: Reference :: Job Server" dependencies { - compile project(path: ":beam-runners-direct-java", configuration: "shadow") - compile project(path: ":beam-runners-java-fn-execution", configuration: "shadow") + compile project(path: ":runners:direct-java", configuration: "shadow") + compile project(path: ":runners:java-fn-execution", configuration: "shadow") compile library.java.slf4j_simple } diff --git a/runners/samza/build.gradle b/runners/samza/build.gradle index 4045b45be7650..5b95105ced5ac 100644 --- a/runners/samza/build.gradle +++ b/runners/samza/build.gradle @@ -28,7 +28,7 @@ description = "Apache Beam :: Runners :: Samza" * the following projects are evaluated before we evaluate this project. This is because * we are attempting to reference the "sourceSets.test.output" directly. */ -evaluationDependsOn(":beam-sdks-java-core") +evaluationDependsOn(":sdks:java:core") configurations { validatesRunner @@ -38,10 +38,10 @@ def samza_version = "1.1.0" dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-runners-core-java", configuration: "shadow") - shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow") - shadow project(path: ":beam-runners-java-fn-execution", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":runners:core-java", configuration: "shadow") + shadow project(path: ":runners:core-construction-java", configuration: "shadow") + shadow project(path: ":runners:java-fn-execution", configuration: "shadow") shadow library.java.jackson_annotations shadow library.java.slf4j_api shadow library.java.joda_time @@ -56,15 +56,15 @@ dependencies { shadow "org.apache.samza:samza-kv-inmemory_2.11:$samza_version" shadow "org.apache.samza:samza-yarn_2.11:$samza_version" shadow "org.apache.kafka:kafka-clients:0.11.0.2" - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") - shadowTest project(path: ":beam-runners-core-java", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") + shadowTest project(path: ":runners:core-java", configuration: "shadowTest") shadowTest library.java.commons_lang3 shadowTest library.java.hamcrest_core shadowTest library.java.junit shadowTest library.java.mockito_core shadowTest library.java.jackson_dataformat_yaml - validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") - validatesRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") + validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest") + validatesRunner project(path: ":runners:core-java", configuration: "shadowTest") validatesRunner project(path: project.path, configuration: "shadow") } @@ -76,7 +76,7 @@ task validatesRunner(type: Test) { ]) classpath = configurations.validatesRunner - testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs) useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' excludeCategories 'org.apache.beam.sdk.testing.LargeKeys$Above100MB' @@ -90,5 +90,5 @@ task validatesRunner(type: Test) { } } -// Generates :beam-runners-samza:runQuickstartJavaSamza +// Generates :runners:samza:runQuickstartJavaSamza createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner:'Samza') diff --git a/runners/samza/job-server/build.gradle b/runners/samza/job-server/build.gradle index 170f6396336e7..d4bc76ca0f72f 100644 --- a/runners/samza/job-server/build.gradle +++ b/runners/samza/job-server/build.gradle @@ -32,7 +32,7 @@ applyJavaNature( ) dependencies { - compile project(path: ":beam-runners-samza", configuration: "shadow") + compile project(path: ":runners:samza", configuration: "shadow") compile group: "org.slf4j", name: "jcl-over-slf4j", version: dependencies.create(project.library.java.slf4j_api).getVersion() compile library.java.slf4j_simple shadow library.java.guava diff --git a/runners/spark/build.gradle b/runners/spark/build.gradle index 5c9ac37b5d09c..b11259c11883d 100644 --- a/runners/spark/build.gradle +++ b/runners/spark/build.gradle @@ -28,8 +28,8 @@ description = "Apache Beam :: Runners :: Spark" * the following projects are evaluated before we evaluate this project. This is because * we are attempting to reference the "sourceSets.test.output" directly. */ -evaluationDependsOn(":beam-sdks-java-core") -evaluationDependsOn(":beam-sdks-java-io-hadoop-format") +evaluationDependsOn(":sdks:java:core") +evaluationDependsOn(":sdks:java:io:hadoop-format") configurations { validatesRunner @@ -54,11 +54,11 @@ test { } dependencies { - shadow project(path: ":beam-model-pipeline", configuration: "shadow") - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow") - shadow project(path: ":beam-runners-core-java", configuration: "shadow") - shadow project(path: ":beam-runners-java-fn-execution", configuration: "shadow") + shadow project(path: ":model:pipeline", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":runners:core-construction-java", configuration: "shadow") + shadow project(path: ":runners:core-java", configuration: "shadow") + shadow project(path: ":runners:java-fn-execution", configuration: "shadow") shadow library.java.guava shadow library.java.jackson_annotations shadow library.java.slf4j_api @@ -79,20 +79,20 @@ dependencies { provided "org.apache.zookeeper:zookeeper:3.4.11" provided "org.scala-lang:scala-library:2.11.8" provided "com.esotericsoftware.kryo:kryo:2.21" - shadowTest project(path: ":beam-sdks-java-io-kafka", configuration: "shadow") - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:io:kafka", configuration: "shadow") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") // SparkStateInternalsTest extends abstract StateInternalsTest - shadowTest project(path: ":beam-runners-core-java", configuration: "shadowTest") - shadowTest project(":beam-sdks-java-harness") + shadowTest project(path: ":runners:core-java", configuration: "shadowTest") + shadowTest project(":sdks:java:harness") shadowTest library.java.avro shadowTest library.java.kafka_clients shadowTest library.java.junit shadowTest library.java.mockito_core shadowTest library.java.jackson_dataformat_yaml shadowTest "org.apache.kafka:kafka_2.11:0.11.0.1" - validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") - validatesRunner project(path: ":beam-sdks-java-io-hadoop-format", configuration: "shadowTest") - validatesRunner project(path: ":beam-examples-java", configuration: "shadowTest") + validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest") + validatesRunner project(path: ":sdks:java:io:hadoop-format", configuration: "shadowTest") + validatesRunner project(path: ":examples:java", configuration: "shadowTest") validatesRunner project(path: project.path, configuration: "shadowTest") validatesRunner project(path: project.path, configuration: "shadow") validatesRunner project(path: project.path, configuration: "provided") @@ -121,7 +121,7 @@ task validatesRunnerBatch(type: Test) { systemProperty "spark.ui.showConsoleProgress", "false" classpath = configurations.validatesRunner - testClassesDirs += files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + testClassesDirs += files(project(":sdks:java:core").sourceSets.test.output.classesDirs) testClassesDirs += files(project.sourceSets.test.output.classesDirs) // Only one SparkContext may be running in a JVM (SPARK-2243) @@ -171,5 +171,5 @@ task validatesRunner { dependsOn validatesRunnerStreaming } -// Generates :beam-runners-spark:runQuickstartJavaSpark +// Generates :runners:spark:runQuickstartJavaSpark createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'Spark') diff --git a/runners/spark/job-server/build.gradle b/runners/spark/job-server/build.gradle index 21587f2b6bbcc..4930e00e80683 100644 --- a/runners/spark/job-server/build.gradle +++ b/runners/spark/job-server/build.gradle @@ -35,7 +35,7 @@ applyJavaNature( }, ) -def sparkRunnerProject = ":${project.name.replace("-job-server", "")}" +def sparkRunnerProject = project.parent.path description = project(sparkRunnerProject).description + " :: Job Server" @@ -52,10 +52,10 @@ dependencies { compile project(path: sparkRunnerProject, configuration: "provided") validatesPortableRunner project(path: sparkRunnerProject, configuration: "shadowTest") validatesPortableRunner project(path: sparkRunnerProject, configuration: "provided") - validatesPortableRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") - validatesPortableRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") - validatesPortableRunner project(path: ":beam-runners-reference-java", configuration: "shadowTest") - compile project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadow") + validatesPortableRunner project(path: ":sdks:java:core", configuration: "shadowTest") + validatesPortableRunner project(path: ":runners:core-java", configuration: "shadowTest") + validatesPortableRunner project(path: ":runners:reference:java", configuration: "shadowTest") + compile project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadow") // TODO: Enable AWS and HDFS file system. } diff --git a/sdks/go/README.md b/sdks/go/README.md index eb822c88f729b..ddbbe9cebaae5 100644 --- a/sdks/go/README.md +++ b/sdks/go/README.md @@ -157,7 +157,7 @@ are added in a PR then the lock file needs to be updated. From the `$GOPATH/src/github.com/apache/beam` directory run ``` -$ ./gradlew :beam-sdks-go:goLock +$ ./gradlew :sdks:go:goLock `./gradlew :goPostcommit` ``` diff --git a/sdks/go/container/build.gradle b/sdks/go/container/build.gradle index 424f1e90c1844..c367d395d788a 100644 --- a/sdks/go/container/build.gradle +++ b/sdks/go/container/build.gradle @@ -25,14 +25,14 @@ description = "Apache Beam :: SDKs :: Go :: Container" // Figure out why the golang plugin does not add a build dependency between projects. // Without the line below, we get spurious errors about not being able to resolve // "./github.com/apache/beam/sdks/go" -resolveBuildDependencies.dependsOn ":beam-sdks-go:goBuild" +resolveBuildDependencies.dependsOn ":sdks:go:goBuild" dependencies { golang { // TODO(herohde): use "./" prefix to prevent gogradle use base github path, for now. // TODO(herohde): get the pkg subdirectory only, if possible. We spend mins pulling cmd/beamctl deps. - build name: './github.com/apache/beam/sdks/go', dir: project(':beam-sdks-go').projectDir - test name: './github.com/apache/beam/sdks/go', dir: project(':beam-sdks-go').projectDir + build name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir + test name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir } } diff --git a/sdks/go/examples/build.gradle b/sdks/go/examples/build.gradle index 70f3338b8d357..f7299f176e7d8 100644 --- a/sdks/go/examples/build.gradle +++ b/sdks/go/examples/build.gradle @@ -30,14 +30,14 @@ def getLocalPlatform = { // Figure out why the golang plugin does not add a build dependency between projects. // Without the line below, we get spurious errors about not being able to resolve // "./github.com/apache/beam/sdks/go" -resolveBuildDependencies.dependsOn ":beam-sdks-go:goBuild" +resolveBuildDependencies.dependsOn ":sdks:go:goBuild" dependencies { golang { // TODO(herohde): use "./" prefix to prevent gogradle use base github path, for now. // TODO(herohde): get the pkg subdirectory only, if possible. We spend mins pulling cmd/beamctl deps. - build name: './github.com/apache/beam/sdks/go', dir: project(':beam-sdks-go').projectDir - test name: './github.com/apache/beam/sdks/go', dir: project(':beam-sdks-go').projectDir + build name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir + test name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir } } diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle index b2f1e2fba6833..67543e8c394f0 100644 --- a/sdks/go/test/build.gradle +++ b/sdks/go/test/build.gradle @@ -24,14 +24,14 @@ description = "Apache Beam :: SDKs :: Go :: Test" // Figure out why the golang plugin does not add a build dependency between projects. // Without the line below, we get spurious errors about not being able to resolve // "./github.com/apache/beam/sdks/go" -resolveBuildDependencies.dependsOn ":beam-sdks-go:goBuild" +resolveBuildDependencies.dependsOn ":sdks:go:goBuild" dependencies { golang { // TODO(herohde): use "./" prefix to prevent gogradle use base github path, for now. // TODO(herohde): get the pkg subdirectory only, if possible. We spend mins pulling cmd/beamctl deps. - build name: './github.com/apache/beam/sdks/go', dir: project(':beam-sdks-go').projectDir - test name: './github.com/apache/beam/sdks/go', dir: project(':beam-sdks-go').projectDir + build name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir + test name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir } } diff --git a/sdks/go/test/run_integration_tests.sh b/sdks/go/test/run_integration_tests.sh index a6e27e629ea81..36fd057a5c307 100755 --- a/sdks/go/test/run_integration_tests.sh +++ b/sdks/go/test/run_integration_tests.sh @@ -63,7 +63,7 @@ fi TAG=$(date +%Y%m%d-%H%M%S) CONTAINER=us.gcr.io/$PROJECT/$USER/go echo "Using container $CONTAINER" -./gradlew :beam-sdks-go-container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG +./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG # Verify it exists docker images | grep $TAG diff --git a/sdks/java/bom/build.gradle b/sdks/java/bom/build.gradle index 288719e450654..9015be15102fc 100644 --- a/sdks/java/bom/build.gradle +++ b/sdks/java/bom/build.gradle @@ -38,17 +38,17 @@ ext { } for (p in rootProject.subprojects) { - if (!p.name.equals(project.name)) { - evaluationDependsOn(':' + p.name) + if (!p.path.equals(project.path)) { + evaluationDependsOn(p.path) } } def bomModuleNames = new ArrayList<>() for (p in rootProject.subprojects) { - def subproject = project(':' + p.name) + def subproject = p // project(':' + p.name) if (subproject.ext.properties.containsKey('includeInJavaBom') && subproject.ext.properties.includeInJavaBom) { - bomModuleNames.add(subproject.name) + bomModuleNames.add(subproject.archivesBaseName) } } @@ -105,6 +105,8 @@ afterEvaluate { publishing { publications { mavenJava(MavenPublication) { + groupId = project.mavenGroupId + artifactId = archivesBaseName version = project.version } } diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle index f7924a0661124..33e094bbe81d3 100644 --- a/sdks/java/container/build.gradle +++ b/sdks/java/container/build.gradle @@ -25,7 +25,7 @@ description = "Apache Beam :: SDKs :: Java :: Container" // Figure out why the golang plugin does not add a build dependency between projects. // Without the line below, we get spurious errors about not being able to resolve // "./github.com/apache/beam/sdks/go" -resolveBuildDependencies.dependsOn ":beam-sdks-go:goBuild" +resolveBuildDependencies.dependsOn ":sdks:go:goBuild" configurations { dockerDependency @@ -35,14 +35,14 @@ dependencies { golang { // TODO(herohde): use "./" prefix to prevent gogradle use base github path, for now. // TODO(herohde): get the pkg subdirectory only, if possible. We spend mins pulling cmd/beamctl deps. - build name: './github.com/apache/beam/sdks/go', dir: project(':beam-sdks-go').projectDir - test name: './github.com/apache/beam/sdks/go', dir: project(':beam-sdks-go').projectDir + build name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir + test name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir } dockerDependency library.java.slf4j_api dockerDependency library.java.slf4j_jdk14 - dockerDependency project(path: ":beam-sdks-java-harness", configuration: "shadow") + dockerDependency project(path: ":sdks:java:harness", configuration: "shadow") // For executing KafkaIO, e.g. as an external transform - dockerDependency project(path: ":beam-sdks-java-io-kafka", configuration: "shadow") + dockerDependency project(path: ":sdks:java:io:kafka", configuration: "shadow") } def dockerfileName = project.findProperty('dockerfile') ?: 'Dockerfile' diff --git a/sdks/java/core/build.gradle b/sdks/java/core/build.gradle index e86b11bfcc1dd..0290683d789b2 100644 --- a/sdks/java/core/build.gradle +++ b/sdks/java/core/build.gradle @@ -60,8 +60,8 @@ test { dependencies { antlr library.java.antlr // Required to load constants from the model, e.g. max timestamp for global window - shadow project(path: ":beam-model-pipeline", configuration: "shadow") - shadow project(path: ":beam-model-job-management", configuration: "shadow") + shadow project(path: ":model:pipeline", configuration: "shadow") + shadow project(path: ":model:job-management", configuration: "shadow") shadow library.java.vendored_guava_20_0 compile library.java.antlr_runtime compile library.java.protobuf_java diff --git a/sdks/java/extensions/euphoria/build.gradle b/sdks/java/extensions/euphoria/build.gradle index 32385a7a16f7b..bd73791e8c9a0 100644 --- a/sdks/java/extensions/euphoria/build.gradle +++ b/sdks/java/extensions/euphoria/build.gradle @@ -22,16 +22,16 @@ applyJavaNature(exportJavadoc: false) description = "Apache Beam :: SDKs :: Java :: Extensions :: Euphoria Java 8 DSL" dependencies { - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.guava testCompile library.java.mockito_core - testCompile project(path: ":beam-sdks-java-extensions-kryo") + testCompile project(path: ":sdks:java:extensions:kryo") testCompile library.java.slf4j_api testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testCompile library.java.mockito_core - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") - testRuntimeOnly project(path: ":beam-runners-direct-java") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") + testRuntimeOnly project(':runners:direct-java') } test { diff --git a/sdks/java/extensions/google-cloud-platform-core/build.gradle b/sdks/java/extensions/google-cloud-platform-core/build.gradle index a089a7743da45..7324fa930308e 100644 --- a/sdks/java/extensions/google-cloud-platform-core/build.gradle +++ b/sdks/java/extensions/google-cloud-platform-core/build.gradle @@ -35,7 +35,7 @@ test { dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.google_http_client_jackson2 shadow library.java.google_auth_library_oauth2_http shadow library.java.google_api_client @@ -51,7 +51,7 @@ dependencies { shadow library.java.jackson_databind provided library.java.hamcrest_core provided library.java.junit - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") testCompile library.java.hamcrest_library testCompile library.java.guava_testlib testCompile library.java.mockito_core diff --git a/sdks/java/extensions/jackson/build.gradle b/sdks/java/extensions/jackson/build.gradle index 6fa170d742c03..e38de09ea5b39 100644 --- a/sdks/java/extensions/jackson/build.gradle +++ b/sdks/java/extensions/jackson/build.gradle @@ -17,6 +17,7 @@ */ plugins { id 'org.apache.beam.module' } +archivesBaseName = 'beam-sdks-java-extensions-json-jackson' applyJavaNature() description = "Apache Beam :: SDKs :: Java :: Extensions :: Jackson" @@ -24,10 +25,10 @@ ext.summary = "Jackson extension provides PTransforms for deserializing and gene dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.jackson_databind testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testCompile library.java.junit - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/extensions/join-library/build.gradle b/sdks/java/extensions/join-library/build.gradle index 12a4ecc194d5a..93b6b4534135a 100644 --- a/sdks/java/extensions/join-library/build.gradle +++ b/sdks/java/extensions/join-library/build.gradle @@ -23,9 +23,9 @@ description = "Apache Beam :: SDKs :: Java :: Extensions :: Join library" dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testCompile library.java.junit - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/extensions/kryo/build.gradle b/sdks/java/extensions/kryo/build.gradle index 21de1321548d5..e949b8bd65253 100644 --- a/sdks/java/extensions/kryo/build.gradle +++ b/sdks/java/extensions/kryo/build.gradle @@ -40,9 +40,9 @@ description = 'Apache Beam :: SDKs :: Java :: Extensions :: Kryo' dependencies { compile "com.esotericsoftware:kryo:${kryoVersion}" - shadow project(path: ':beam-sdks-java-core', configuration: 'shadow') - testCompile project(path: ':beam-sdks-java-core', configuration: 'shadowTest') - testRuntimeOnly project(path: ":beam-runners-direct-java") + shadow project(path: ':sdks:java:core', configuration: 'shadow') + testCompile project(path: ':sdks:java:core', configuration: 'shadowTest') + testRuntimeOnly project(':runners:direct-java') } test { diff --git a/sdks/java/extensions/protobuf/build.gradle b/sdks/java/extensions/protobuf/build.gradle index 7897256d900d1..ece59da080d0a 100644 --- a/sdks/java/extensions/protobuf/build.gradle +++ b/sdks/java/extensions/protobuf/build.gradle @@ -25,9 +25,9 @@ ext.summary = "Add support to Apache Beam for Google Protobuf." dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.protobuf_java - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testCompile library.java.mockito_core diff --git a/sdks/java/extensions/sketching/build.gradle b/sdks/java/extensions/sketching/build.gradle index c53fea8498cbd..7932d0797d130 100644 --- a/sdks/java/extensions/sketching/build.gradle +++ b/sdks/java/extensions/sketching/build.gradle @@ -26,15 +26,15 @@ def tdigest_version = "3.2" dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow "com.clearspring.analytics:stream:$streamlib_version" shadow "com.tdunning:t-digest:$tdigest_version" shadow library.java.slf4j_api shadowTest library.java.avro shadowTest library.java.commons_lang3 - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") shadowTest library.java.hamcrest_core shadowTest library.java.hamcrest_library shadowTest library.java.junit - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/extensions/sorter/build.gradle b/sdks/java/extensions/sorter/build.gradle index 8d464b152779d..7f0c72c9d637e 100644 --- a/sdks/java/extensions/sorter/build.gradle +++ b/sdks/java/extensions/sorter/build.gradle @@ -22,7 +22,7 @@ applyJavaNature() description = "Apache Beam :: SDKs :: Java :: Extensions :: Sorter" dependencies { - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.vendored_guava_20_0 provided library.java.hadoop_mapreduce_client_core provided library.java.hadoop_common @@ -30,5 +30,5 @@ dependencies { testCompile library.java.hamcrest_library testCompile library.java.mockito_core testCompile library.java.junit - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 6de77bd912703..700398819b2f7 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -71,8 +71,8 @@ dependencies { compile "org.apache.calcite:calcite-core:$calcite_version" compile "org.apache.calcite:calcite-linq4j:$calcite_version" compile "org.apache.calcite.avatica:avatica-core:$avatica_version" - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-extensions-join-library", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":sdks:java:extensions:join-library", configuration: "shadow") shadow library.java.slf4j_api shadow library.java.commons_codec shadow library.java.commons_csv @@ -82,9 +82,9 @@ dependencies { shadow library.java.joda_time shadow "com.alibaba:fastjson:1.2.49" shadow "com.jayway.jsonpath:json-path:2.4.0" - shadow project(path: ":beam-runners-direct-java", configuration: "shadow") - provided project(path: ":beam-sdks-java-io-kafka", configuration: "shadow") - provided project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") + shadow project(path: ":runners:direct-java", configuration: "shadow") + provided project(path: ":sdks:java:io:kafka", configuration: "shadow") + provided project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow") provided library.java.kafka_clients shadowTest library.java.junit shadowTest library.java.hamcrest_core @@ -190,11 +190,11 @@ task integrationTest(type: Test) { include '**/*IT.class' maxParallelForks 4 - classpath = project(":beam-sdks-java-extensions-sql") + classpath = project(":sdks:java:extensions:sql") .sourceSets .test .runtimeClasspath - testClassesDirs = files(project(":beam-sdks-java-extensions-sql").sourceSets.test.output.classesDirs) + testClassesDirs = files(project(":sdks:java:extensions:sql").sourceSets.test.output.classesDirs) useJUnit { } } diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index 821d28a9472f7..3496a9e01874f 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -27,7 +27,7 @@ applyJavaNature( dependencies { shadow library.java.grpc_google_cloud_datacatalog_v1beta1 shadow library.java.proto_google_cloud_datacatalog_v1beta1 - provided project(path: ":beam-sdks-java-extensions-sql", configuration: "shadow") + provided project(path: ":sdks:java:extensions:sql", configuration: "shadow") // For Data Catalog GRPC client provided library.java.grpc_all @@ -36,7 +36,7 @@ dependencies { provided library.java.netty_tcnative_boringssl_static // Dependencies for the example - provided project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") + provided project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow") provided library.java.vendored_guava_20_0 provided library.java.slf4j_api shadowTestRuntimeClasspath library.java.slf4j_simple diff --git a/sdks/java/extensions/sql/hcatalog/build.gradle b/sdks/java/extensions/sql/hcatalog/build.gradle index 1aedb02bfd5b4..9a15eec336eac 100644 --- a/sdks/java/extensions/sql/hcatalog/build.gradle +++ b/sdks/java/extensions/sql/hcatalog/build.gradle @@ -26,8 +26,8 @@ def hive_version = "2.1.0" def netty_version = "4.1.30.Final" dependencies { - provided project(path: ":beam-sdks-java-extensions-sql", configuration: "shadow") - provided project(path: ":beam-sdks-java-io-hcatalog", configuration: "shadow") + provided project(path: ":sdks:java:extensions:sql", configuration: "shadow") + provided project(path: ":sdks:java:io:hcatalog", configuration: "shadow") // Needed for HCatalogTableProvider tests, // they use HCat* types diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index e61c12572bde9..70cbad1d7829d 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -31,12 +31,12 @@ configurations { } dependencies { - compile project(path: ":beam-sdks-java-extensions-sql", configuration: "shadow") + compile project(path: ":sdks:java:extensions:sql", configuration: "shadow") compile "jline:jline:2.14.6" compile "sqlline:sqlline:1.4.0" compile library.java.slf4j_jdk14 compile library.java.guava - testCompile project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadowTest") testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library @@ -66,7 +66,7 @@ test { } } -task shadowJarTest(type: Test, dependsOn: ":beam-sdks-java-extensions-sql-jdbc:shadowJar") { +task shadowJarTest(type: Test, dependsOn: ":sdks:java:extensions:sql:jdbc:shadowJar") { group = "Verification" // Disable Gradle cache (it should not be used because the IT's won't run). @@ -96,8 +96,8 @@ task endToEndTest(type: Test) { systemProperty "beamTestPipelineOptions", JsonOutput.toJson(pipelineOptions) include '**/*IT.class' - classpath = project(":beam-sdks-java-extensions-sql-jdbc").sourceSets.test.runtimeClasspath - testClassesDirs = files(project(":beam-sdks-java-extensions-sql-jdbc").sourceSets.test.output.classesDirs) + classpath = project(":sdks:java:extensions:sql:jdbc").sourceSets.test.runtimeClasspath + testClassesDirs = files(project(":sdks:java:extensions:sql:jdbc").sourceSets.test.output.classesDirs) useJUnit { } } diff --git a/sdks/java/extensions/sql/shell/build.gradle b/sdks/java/extensions/sql/shell/build.gradle index 3d04ee2e075ed..50d281f573a10 100644 --- a/sdks/java/extensions/sql/shell/build.gradle +++ b/sdks/java/extensions/sql/shell/build.gradle @@ -22,8 +22,8 @@ plugins { } dependencies { - compile project(path: ":beam-sdks-java-extensions-sql-jdbc", configuration: "shadow") - permitUnusedDeclared project(path: ":beam-sdks-java-extensions-sql-jdbc", configuration: "shadow") + compile project(path: ":sdks:java:extensions:sql:jdbc", configuration: "shadow") + permitUnusedDeclared project(path: ":sdks:java:extensions:sql:jdbc", configuration: "shadow") if (project.hasProperty("beam.sql.shell.bundled")) { project.getProperty("beam.sql.shell.bundled").tokenize(",").each { @@ -52,7 +52,7 @@ mainClassName = "org.apache.beam.sdk.extensions.sql.jdbc.BeamSqlLine" * So, correct gradle invocation to start the SQL shell * looks something like this: * - * ./gradlew :beam-sdks-java-extensions-sql-shell:run --console plain -q --no-daemon + * ./gradlew :sdks:java:extensions:sql:shell:run --console plain -q --no-daemon */ run { standardInput System.in diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java index 70867508e9298..a4f697d63800f 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java @@ -39,7 +39,7 @@ *

Run the example from the Beam source root with * * 

- *   ./gradlew :beam-sdks-java-extensions-sql:runBasicExample
+ *   ./gradlew :sdks:java:extensions:sql:runBasicExample
  *
* *

The above command executes the example locally using direct runner. Running the pipeline in diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlPojoExample.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlPojoExample.java index 97f80c5d565f2..7e5995bff47cd 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlPojoExample.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlPojoExample.java @@ -38,7 +38,7 @@ *

Run the example from the Beam source root with * * 

- *   ./gradlew :beam-sdks-java-extensions-sql:runPojoExample
+ *   ./gradlew :sdks:java:extensions:sql:runPojoExample
  *
* *

The above command executes the example locally using direct runner. Running the pipeline in diff --git a/sdks/java/fn-execution/build.gradle b/sdks/java/fn-execution/build.gradle index 7dce8693cc252..eb7da2baf7b75 100644 --- a/sdks/java/fn-execution/build.gradle +++ b/sdks/java/fn-execution/build.gradle @@ -24,9 +24,9 @@ ext.summary = """Contains code shared across the Beam Java SDK Harness and Java the Beam Portability Framework.""" dependencies { - shadow project(path: ":beam-model-pipeline", configuration: "shadow") - shadow project(path: ":beam-model-fn-execution", configuration: "shadow") - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":model:pipeline", configuration: "shadow") + shadow project(path: ":model:fn-execution", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.vendored_grpc_1_13_1 shadow library.java.vendored_guava_20_0 shadow library.java.slf4j_api diff --git a/sdks/java/harness/build.gradle b/sdks/java/harness/build.gradle index ced8785566d35..3c107e8acd190 100644 --- a/sdks/java/harness/build.gradle +++ b/sdks/java/harness/build.gradle @@ -21,10 +21,10 @@ plugins { id 'org.apache.beam.module' } // We specifically enumerate all the projects that we depend on since // the list is used in both defining the included set for the uber jar // and also the set of project level dependencies. -def dependOnProjects = [":beam-model-pipeline", ":beam-model-fn-execution", ":beam-sdks-java-core", - ":beam-sdks-java-fn-execution", - ":beam-sdks-java-extensions-google-cloud-platform-core", - ":beam-runners-core-java", ":beam-runners-core-construction-java"] +def dependOnProjects = [":model:pipeline", ":model:fn-execution", ":sdks:java:core", + ":sdks:java:fn-execution", + ":sdks:java:extensions:google-cloud-platform-core", + ":runners:core-java", ":runners:core-construction-java"] applyJavaNature( validateShadowJar: false, @@ -63,6 +63,6 @@ dependencies { testCompile library.java.hamcrest_library testCompile library.java.junit testCompile library.java.mockito_core - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") shadowTestRuntimeClasspath library.java.slf4j_jdk14 } diff --git a/sdks/java/io/amazon-web-services/build.gradle b/sdks/java/io/amazon-web-services/build.gradle index 4711f40a2ffc5..c39ca4d255d39 100644 --- a/sdks/java/io/amazon-web-services/build.gradle +++ b/sdks/java/io/amazon-web-services/build.gradle @@ -26,7 +26,7 @@ ext.summary = "IO library to read and write Amazon Web Services services from Be dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.aws_java_sdk_cloudwatch shadow library.java.aws_java_sdk_core shadow library.java.aws_java_sdk_s3 @@ -39,8 +39,8 @@ dependencies { shadow library.java.slf4j_api runtime 'commons-codec:commons-codec:1.9' runtime "org.apache.httpcomponents:httpclient:4.5.6" - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") testCompile "io.findify:s3mock_2.12:0.2.4" shadowTest library.java.guava_testlib shadowTest library.java.hamcrest_core @@ -49,7 +49,7 @@ dependencies { shadowTest library.java.junit shadowTest group: 'org.elasticmq', name: 'elasticmq-rest-sqs_2.12', version: '0.14.1' testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } test { diff --git a/sdks/java/io/amqp/build.gradle b/sdks/java/io/amqp/build.gradle index facc202e40c43..f74033b7f7e97 100644 --- a/sdks/java/io/amqp/build.gradle +++ b/sdks/java/io/amqp/build.gradle @@ -24,7 +24,7 @@ ext.summary = "IO to read and write using AMQP 1.0 protocol (http://www.amqp.org dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.joda_time shadow "org.apache.qpid:proton-j:0.13.1" testCompile library.java.slf4j_api @@ -35,5 +35,5 @@ dependencies { testCompile library.java.activemq_amqp testCompile library.java.activemq_junit testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/cassandra/build.gradle b/sdks/java/io/cassandra/build.gradle index ed0fc605323dd..db3896fa5630e 100644 --- a/sdks/java/io/cassandra/build.gradle +++ b/sdks/java/io/cassandra/build.gradle @@ -47,11 +47,11 @@ def achilles_version = "6.0.2" dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.slf4j_api shadow library.java.cassandra_driver_core shadow library.java.cassandra_driver_mapping - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library @@ -61,5 +61,5 @@ dependencies { testCompile group: 'info.archinnov', name: 'achilles-junit', version: "$achilles_version" testCompile library.java.jackson_jaxb_annotations testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/clickhouse/build.gradle b/sdks/java/io/clickhouse/build.gradle index 739eecbf0b9ff..062d1972acc30 100644 --- a/sdks/java/io/clickhouse/build.gradle +++ b/sdks/java/io/clickhouse/build.gradle @@ -50,7 +50,7 @@ def testcontainers_version = "1.10.5" dependencies { javacc "net.java.dev.javacc:javacc:4.0" - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.joda_time shadow "ru.yandex.clickhouse:clickhouse-jdbc:$clickhouse_jdbc_version" testCompile library.java.slf4j_api @@ -59,5 +59,5 @@ dependencies { testCompile library.java.hamcrest_library testCompile "org.testcontainers:clickhouse:$testcontainers_version" testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/common/build.gradle b/sdks/java/io/common/build.gradle index d7e99c8672eb4..f137578a259b9 100644 --- a/sdks/java/io/common/build.gradle +++ b/sdks/java/io/common/build.gradle @@ -26,5 +26,5 @@ dependencies { shadowTest library.java.vendored_guava_20_0 testCompile library.java.junit testCompile library.java.postgres - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadow") + shadowTest project(path: ":sdks:java:core", configuration: "shadow") } diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/build.gradle b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/build.gradle index 17cce8ce10cd1..05578ecf8d3c9 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/build.gradle +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/build.gradle @@ -17,6 +17,7 @@ */ plugins { id 'org.apache.beam.module' } +archivesBaseName = 'beam-sdks-java-io-elasticsearch-tests-2' applyJavaNature() provideIntegrationTestingDependencies() enableJavaPerformanceTesting() @@ -29,10 +30,10 @@ def log4j_version = "2.6.2" def elastic_search_version = "2.4.1" dependencies { - testCompile project(path: ":beam-sdks-java-io-elasticsearch-tests-common", configuration: "shadowTest") - testCompile project(path: ":beam-sdks-java-core", configuration: "shadow") - testCompile project(path: ":beam-sdks-java-io-elasticsearch", configuration: "shadow") - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:elasticsearch-tests:elasticsearch-tests-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadow") + testCompile project(path: ":sdks:java:io:elasticsearch", configuration: "shadow") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") testCompile library.java.slf4j_api testCompile "net.java.dev.jna:jna:$jna_version" testCompile "org.apache.logging.log4j:log4j-api:$log4j_version" @@ -45,5 +46,5 @@ dependencies { shadowTest library.java.vendored_guava_20_0 testCompile "org.elasticsearch:elasticsearch:$elastic_search_version" testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/build.gradle b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/build.gradle index bb2237e9ef744..025470c4ffc5e 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/build.gradle +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/build.gradle @@ -17,6 +17,7 @@ */ plugins { id 'org.apache.beam.module' } +archivesBaseName = 'beam-sdks-java-io-elasticsearch-tests-5' applyJavaNature() provideIntegrationTestingDependencies() enableJavaPerformanceTesting() @@ -43,15 +44,15 @@ configurations.all { } dependencies { - testCompile project(path: ":beam-sdks-java-io-elasticsearch-tests-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:elasticsearch-tests:elasticsearch-tests-common", configuration: "shadowTest") testCompile "org.elasticsearch.test:framework:$elastic_search_version" testCompile "org.elasticsearch.plugin:transport-netty4-client:$elastic_search_version" testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.5.0" testCompile "org.elasticsearch:elasticsearch:$elastic_search_version" - testCompile project(path: ":beam-sdks-java-core", configuration: "shadow") - testCompile project(path: ":beam-sdks-java-io-elasticsearch", configuration: "shadow") - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadow") + testCompile project(path: ":sdks:java:io:elasticsearch", configuration: "shadow") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") testCompile "org.apache.logging.log4j:log4j-core:$log4j_version" testCompile "org.apache.logging.log4j:log4j-api:$log4j_version" testCompile library.java.slf4j_api @@ -62,5 +63,5 @@ dependencies { testCompile library.java.junit testCompile "org.elasticsearch.client:elasticsearch-rest-client:$elastic_search_version" testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/build.gradle b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/build.gradle index 359450328d4fe..0b269ffed5f86 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/build.gradle +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/build.gradle @@ -17,6 +17,7 @@ */ plugins { id 'org.apache.beam.module' } +archivesBaseName = 'beam-sdks-java-io-elasticsearch-tests-6' applyJavaNature() provideIntegrationTestingDependencies() enableJavaPerformanceTesting() @@ -43,15 +44,15 @@ configurations.all { } dependencies { - testCompile project(path: ":beam-sdks-java-io-elasticsearch-tests-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:elasticsearch-tests:elasticsearch-tests-common", configuration: "shadowTest") testCompile "org.elasticsearch.test:framework:$elastic_search_version" testCompile "org.elasticsearch.plugin:transport-netty4-client:$elastic_search_version" testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.5.2" testCompile "org.elasticsearch:elasticsearch:$elastic_search_version" - testCompile project(path: ":beam-sdks-java-core", configuration: "shadow") - testCompile project(path: ":beam-sdks-java-io-elasticsearch", configuration: "shadow") - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadow") + testCompile project(path: ":sdks:java:io:elasticsearch", configuration: "shadow") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") testCompile "org.apache.logging.log4j:log4j-core:$log4j_version" testCompile "org.apache.logging.log4j:log4j-api:$log4j_version" testCompile library.java.slf4j_api @@ -62,5 +63,5 @@ dependencies { testCompile library.java.junit testCompile "org.elasticsearch.client:elasticsearch-rest-client:$elastic_search_version" testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/build.gradle b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/build.gradle index 2e9a4b3ecf101..29374fe4867b5 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/build.gradle +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/build.gradle @@ -17,6 +17,7 @@ */ plugins { id 'org.apache.beam.module' } +archivesBaseName = 'beam-sdks-java-io-elasticsearch-tests-common' applyJavaNature() description = "Apache Beam :: SDKs :: Java :: IO :: Elasticsearch-Tests :: Common" @@ -32,9 +33,9 @@ dependencies { testCompile "org.apache.httpcomponents:httpcore:4.4.10" testCompile "org.apache.httpcomponents:httpclient:4.5.6" - testCompile project(path: ":beam-sdks-java-core", configuration: "shadow") - testCompile project(path: ":beam-sdks-java-io-elasticsearch", configuration: "shadow") - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadow") + testCompile project(path: ":sdks:java:io:elasticsearch", configuration: "shadow") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") testCompile library.java.slf4j_api testCompile "net.java.dev.jna:jna:$jna_version" testCompile "org.apache.logging.log4j:log4j-api:$log4j_version" @@ -45,5 +46,5 @@ dependencies { testCompile library.java.junit testCompile "org.elasticsearch.client:elasticsearch-rest-client:6.4.0" testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/elasticsearch/build.gradle b/sdks/java/io/elasticsearch/build.gradle index f490115db1af8..196222c4d0e9b 100644 --- a/sdks/java/io/elasticsearch/build.gradle +++ b/sdks/java/io/elasticsearch/build.gradle @@ -24,7 +24,7 @@ ext.summary = "IO to read and write on Elasticsearch" dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.jackson_databind shadow library.java.jackson_annotations shadow "org.elasticsearch.client:elasticsearch-rest-client:6.4.0" @@ -32,5 +32,5 @@ dependencies { shadow "org.apache.httpcomponents:httpcore-nio:4.4.10" shadow "org.apache.httpcomponents:httpcore:4.4.10" shadow "org.apache.httpcomponents:httpclient:4.5.6" - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") } diff --git a/sdks/java/io/file-based-io-tests/build.gradle b/sdks/java/io/file-based-io-tests/build.gradle index 1d74da3ff2b3f..26c10a50afcd2 100644 --- a/sdks/java/io/file-based-io-tests/build.gradle +++ b/sdks/java/io/file-based-io-tests/build.gradle @@ -25,11 +25,11 @@ description = "Apache Beam :: SDKs :: Java :: File-based-io-tests" ext.summary = "Integration tests for reading/writing using file-based sources/sinks." dependencies { - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-io-xml", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-io-parquet", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-test-utils", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:io:common", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:io:xml", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:io:parquet", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:testing:test-utils", configuration: "shadowTest") shadowTest library.java.guava shadowTest library.java.junit shadowTest library.java.hamcrest_core diff --git a/sdks/java/io/google-cloud-platform/build.gradle b/sdks/java/io/google-cloud-platform/build.gradle index 4eb0b2ee1eb2f..57ffe214c2078 100644 --- a/sdks/java/io/google-cloud-platform/build.gradle +++ b/sdks/java/io/google-cloud-platform/build.gradle @@ -33,9 +33,9 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Google Cloud Platform" ext.summary = "IO library to read and write Google Cloud Platform systems from Beam." dependencies { - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-extensions-protobuf", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadow") + shadow project(path: ":sdks:java:extensions:protobuf", configuration: "shadow") shadow library.java.guava shadow library.java.jackson_databind shadow library.java.grpc_core @@ -71,9 +71,9 @@ dependencies { shadow library.java.proto_google_common_protos shadow library.java.grpc_all shadow library.java.netty_tcnative_boringssl_static - shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadowTest") - shadowTest project(path: ":beam-runners-direct-java", configuration: "shadow") + shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadowTest") + shadowTest project(path: ":runners:direct-java", configuration: "shadow") shadowTest library.java.hamcrest_core shadowTest library.java.hamcrest_library shadowTest library.java.mockito_core diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java index bee548ec7166e..1733334f1071c 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java @@ -48,7 +48,7 @@ import org.junit.runners.JUnit4; // Run a specific test using: -// ./gradlew :beam-sdks-java-io-google-cloud-platform:integrationTest --tests +// ./gradlew :sdks:java:io:google-cloud-platform:integrationTest --tests // GcsKmsKeyIT.testGcsWriteWithKmsKey --info /** Integration test for GCS CMEK support. */ diff --git a/sdks/java/io/hadoop-common/build.gradle b/sdks/java/io/hadoop-common/build.gradle index 6c1b58f536688..c77235efcf07c 100644 --- a/sdks/java/io/hadoop-common/build.gradle +++ b/sdks/java/io/hadoop-common/build.gradle @@ -23,7 +23,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Hadoop Common" ext.summary = "Library to add shared Hadoop classes among Beam IOs." dependencies { - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") provided library.java.hadoop_client provided library.java.hadoop_common provided library.java.hadoop_mapreduce_client_core diff --git a/sdks/java/io/hadoop-file-system/build.gradle b/sdks/java/io/hadoop-file-system/build.gradle index cb4f7290ec305..a4e5c70dfd839 100644 --- a/sdks/java/io/hadoop-file-system/build.gradle +++ b/sdks/java/io/hadoop-file-system/build.gradle @@ -24,14 +24,14 @@ ext.summary = "Library to read and write Hadoop/HDFS file formats from Beam." dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.jackson_core shadow library.java.jackson_databind shadow library.java.slf4j_api provided library.java.hadoop_client provided library.java.hadoop_common provided library.java.hadoop_mapreduce_client_core - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") testCompile library.java.guava_testlib testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library @@ -39,6 +39,6 @@ dependencies { testCompile library.java.junit testCompile library.java.hadoop_minicluster testCompile library.java.hadoop_hdfs_tests - testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly library.java.slf4j_jdk14 + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/hadoop-format/build.gradle b/sdks/java/io/hadoop-format/build.gradle index 1d9f36cf35557..9e7dba7cd0bc6 100644 --- a/sdks/java/io/hadoop-format/build.gradle +++ b/sdks/java/io/hadoop-format/build.gradle @@ -44,18 +44,18 @@ configurations.testRuntimeClasspath { } dependencies { - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.vendored_guava_20_0 shadow library.java.slf4j_api - shadow project(path: ":beam-sdks-java-io-hadoop-common", configuration: "shadow") + shadow project(path: ":sdks:java:io:hadoop-common", configuration: "shadow") provided library.java.hadoop_common provided library.java.hadoop_hdfs provided library.java.hadoop_mapreduce_client_core - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-test-utils", configuration: "shadowTest") - testCompile project(path: ":beam-sdks-java-io-jdbc", configuration: "shadow") - testCompile project(path: ":beam-examples-java", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:testing:test-utils", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:jdbc", configuration: "shadow") + testCompile project(path: ":examples:java", configuration: "shadowTest") testCompile "org.elasticsearch.plugin:transport-netty4-client:$elastic_search_version" testCompile "org.elasticsearch.client:transport:$elastic_search_version" @@ -82,15 +82,15 @@ dependencies { testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") shadow library.java.commons_io_2x - delegate.add("sparkRunner", project(path: ":beam-sdks-java-io-hadoop-format", configuration: "shadow")) - delegate.add("sparkRunner", project(path: ":beam-sdks-java-io-hadoop-format", configuration: "shadowTest")) + delegate.add("sparkRunner", project(path: ":sdks:java:io:hadoop-format", configuration: "shadow")) + delegate.add("sparkRunner", project(path: ":sdks:java:io:hadoop-format", configuration: "shadowTest")) - sparkRunner project(path: ":beam-examples-java", configuration: "shadowTest") - sparkRunner project(path: ":beam-runners-spark", configuration: "shadow") - sparkRunner project(path: ":beam-sdks-java-io-hadoop-file-system", configuration: "shadow") + sparkRunner project(path: ":examples:java", configuration: "shadowTest") + sparkRunner project(path: ":runners:spark", configuration: "shadow") + sparkRunner project(path: ":sdks:java:io:hadoop-file-system", configuration: "shadow") sparkRunner library.java.spark_streaming sparkRunner library.java.spark_core } diff --git a/sdks/java/io/hbase/build.gradle b/sdks/java/io/hbase/build.gradle index 2469a5b085d7f..4588a9fb7276c 100644 --- a/sdks/java/io/hbase/build.gradle +++ b/sdks/java/io/hbase/build.gradle @@ -38,12 +38,12 @@ def hbase_version = "1.2.6" dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-io-hadoop-common", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":sdks:java:io:hadoop-common", configuration: "shadow") shadow library.java.slf4j_api shadow "org.apache.hbase:hbase-shaded-client:$hbase_version" - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") testCompile library.java.commons_lang3 testCompile library.java.junit testCompile library.java.hamcrest_core @@ -64,6 +64,6 @@ dependencies { } testCompile "org.apache.hbase:hbase-hadoop-compat:$hbase_version:tests" testCompile "org.apache.hbase:hbase-hadoop2-compat:$hbase_version:tests" - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/hcatalog/build.gradle b/sdks/java/io/hcatalog/build.gradle index 575348617aa42..d48f18aa837db 100644 --- a/sdks/java/io/hcatalog/build.gradle +++ b/sdks/java/io/hcatalog/build.gradle @@ -38,12 +38,12 @@ test { * TODO: Swap to generating test artifacts which we can then rely on instead of * the test outputs directly. */ -evaluationDependsOn(":beam-sdks-java-io-common") +evaluationDependsOn(":sdks:java:io:common") dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-sdks-java-io-hadoop-common", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":sdks:java:io:hadoop-common", configuration: "shadow") shadow library.java.slf4j_api // Hive bundles without repackaging Jackson which is why we redeclare it here so that it appears // on the compile/test/runtime classpath before Hive. @@ -59,7 +59,7 @@ dependencies { exclude group: "org.apache.hive", module: "hive-exec" exclude group: "com.google.protobuf", module: "protobuf-java" } - testCompile project(":beam-sdks-java-io-common").sourceSets.test.output + testCompile project(":sdks:java:io:common").sourceSets.test.output testCompile library.java.commons_io_2x testCompile library.java.junit testCompile library.java.hamcrest_core @@ -68,6 +68,6 @@ dependencies { testCompile "org.apache.hive:hive-exec:$hive_version" testCompile "org.apache.hive:hive-common:$hive_version" testCompile "org.apache.hive:hive-cli:$hive_version" - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/jdbc/build.gradle b/sdks/java/io/jdbc/build.gradle index 4e715921e383c..ca7bb2a43660d 100644 --- a/sdks/java/io/jdbc/build.gradle +++ b/sdks/java/io/jdbc/build.gradle @@ -26,11 +26,11 @@ ext.summary = "IO to read and write on JDBC datasource." dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow "org.apache.commons:commons-dbcp2:2.6.0" - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-test-utils", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:testing:test-utils", configuration: "shadowTest") testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library @@ -40,5 +40,5 @@ dependencies { testCompile group: "org.apache.derby", name: "derbyclient", version:"10.14.2.0" testCompile group: "org.apache.derby", name: "derbynet", version:"10.14.2.0" testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/jms/build.gradle b/sdks/java/io/jms/build.gradle index 7b39f08130494..fdb8c7e5462eb 100644 --- a/sdks/java/io/jms/build.gradle +++ b/sdks/java/io/jms/build.gradle @@ -25,7 +25,7 @@ destinations (queues and topics).""" dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.slf4j_api shadow library.java.joda_time shadow "org.apache.geronimo.specs:geronimo-jms_1.1_spec:1.1.1" @@ -37,5 +37,5 @@ dependencies { testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/kafka/build.gradle b/sdks/java/io/kafka/build.gradle index 8c194c6c1e36c..3a1db4b24750c 100644 --- a/sdks/java/io/kafka/build.gradle +++ b/sdks/java/io/kafka/build.gradle @@ -24,20 +24,20 @@ ext.summary = "Library to read Kafka topics." dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.kafka_clients shadow library.java.slf4j_api shadow library.java.joda_time shadow library.java.jackson_annotations shadow library.java.jackson_databind shadow "org.springframework:spring-expression:4.3.18.RELEASE" - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") // For testing Cross-language transforms - testCompile project(path: ":beam-runners-core-construction-java", configuration: "shadow") + testCompile project(path: ":runners:core-construction-java", configuration: "shadow") testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testCompile library.java.junit testCompile library.java.powermock testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/kinesis/build.gradle b/sdks/java/io/kinesis/build.gradle index 7313ed881b8e0..be21f3a14ed20 100644 --- a/sdks/java/io/kinesis/build.gradle +++ b/sdks/java/io/kinesis/build.gradle @@ -38,7 +38,7 @@ test { } dependencies { - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.guava shadow library.java.slf4j_api shadow library.java.joda_time @@ -56,5 +56,5 @@ dependencies { testCompile library.java.hamcrest_library testCompile "org.assertj:assertj-core:3.11.1" testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/kudu/build.gradle b/sdks/java/io/kudu/build.gradle index 4e5601b5b9610..4acc7cdac1f00 100644 --- a/sdks/java/io/kudu/build.gradle +++ b/sdks/java/io/kudu/build.gradle @@ -36,15 +36,15 @@ def kudu_version = "1.9.0" dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow "org.apache.kudu:kudu-client:$kudu_version" shadow library.java.slf4j_api - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testCompile library.java.junit testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/mongodb/build.gradle b/sdks/java/io/mongodb/build.gradle index 645d902f1e451..ad46ca94a12d0 100644 --- a/sdks/java/io/mongodb/build.gradle +++ b/sdks/java/io/mongodb/build.gradle @@ -26,17 +26,17 @@ ext.summary = "IO to read and write on MongoDB." dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.slf4j_api shadow library.java.joda_time shadow "org.mongodb:mongo-java-driver:3.9.1" testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") - shadowTest project(path: ":beam-sdks-java-test-utils", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") + shadowTest project(path: ":sdks:java:testing:test-utils", configuration: "shadowTest") testCompile "de.flapdoodle.embed:de.flapdoodle.embed.mongo:2.2.0" testCompile "de.flapdoodle.embed:de.flapdoodle.embed.process:2.1.2" testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/mqtt/build.gradle b/sdks/java/io/mqtt/build.gradle index 6c63a4685b0a2..e27952265bc52 100644 --- a/sdks/java/io/mqtt/build.gradle +++ b/sdks/java/io/mqtt/build.gradle @@ -24,12 +24,12 @@ ext.summary = "IO to read and write to a MQTT broker." dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.slf4j_api shadow library.java.joda_time shadow "org.fusesource.mqtt-client:mqtt-client:1.15" shadow "org.fusesource.hawtbuf:hawtbuf:1.11" - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") testCompile library.java.activemq_broker testCompile library.java.activemq_mqtt testCompile library.java.activemq_kahadb_store @@ -37,5 +37,5 @@ dependencies { testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/parquet/build.gradle b/sdks/java/io/parquet/build.gradle index 588bcac26a8f5..fe0f28d963d02 100644 --- a/sdks/java/io/parquet/build.gradle +++ b/sdks/java/io/parquet/build.gradle @@ -26,7 +26,7 @@ def parquet_version = "1.10.0" dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.slf4j_api shadow "org.apache.parquet:parquet-avro:$parquet_version" shadow "org.apache.parquet:parquet-common:$parquet_version" @@ -34,10 +34,10 @@ dependencies { shadow library.java.avro shadow library.java.hadoop_client shadow library.java.hadoop_common - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/rabbitmq/build.gradle b/sdks/java/io/rabbitmq/build.gradle index 9900affb6da4f..a890edcc652f3 100644 --- a/sdks/java/io/rabbitmq/build.gradle +++ b/sdks/java/io/rabbitmq/build.gradle @@ -24,11 +24,10 @@ ext.summary = "IO to read and write to a RabbitMQ broker." dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.joda_time shadow "com.rabbitmq:amqp-client:4.9.3" - testCompile project(path: ":beam-runners-direct-java", configuration: "shadow") - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") testCompile "org.apache.qpid:qpid-broker:0.28" testCompile "org.apache.qpid:qpid-broker-core:0.28" testCompile library.java.junit @@ -36,5 +35,5 @@ dependencies { testCompile library.java.hamcrest_library testCompile library.java.slf4j_api testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/redis/build.gradle b/sdks/java/io/redis/build.gradle index 879665462f8fd..f8b4112ad3f40 100644 --- a/sdks/java/io/redis/build.gradle +++ b/sdks/java/io/redis/build.gradle @@ -24,13 +24,13 @@ ext.summary ="IO to read and write on a Redis keystore." dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow "redis.clients:jedis:3.0.1" - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadowTest") testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testCompile "com.github.kstyrc:embedded-redis:0.6" testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/solr/build.gradle b/sdks/java/io/solr/build.gradle index 137a909daf366..6ce544e63adf8 100644 --- a/sdks/java/io/solr/build.gradle +++ b/sdks/java/io/solr/build.gradle @@ -24,12 +24,12 @@ ext.summary = "IO to read and write from/to Solr." dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.commons_compress shadow "org.apache.solr:solr-solrj:5.5.4" compileOnly "org.apache.httpcomponents:httpclient:4.5.6" - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") - testCompile project(path: ":beam-sdks-java-io-common", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:io:common", configuration: "shadow") testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testCompile library.java.junit @@ -38,5 +38,5 @@ dependencies { testCompile "org.apache.solr:solr-core:5.5.4" testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.3.2" testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/synthetic/build.gradle b/sdks/java/io/synthetic/build.gradle index 9f0a754b46abf..57019b1e65dc3 100644 --- a/sdks/java/io/synthetic/build.gradle +++ b/sdks/java/io/synthetic/build.gradle @@ -34,6 +34,6 @@ dependencies { testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - testRuntimeOnly project(path: ":beam-runners-direct-java") + shadow project(path: ":sdks:java:core", configuration: "shadow") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/tika/build.gradle b/sdks/java/io/tika/build.gradle index 8c130feca8bac..9fba673e252b6 100644 --- a/sdks/java/io/tika/build.gradle +++ b/sdks/java/io/tika/build.gradle @@ -28,13 +28,13 @@ def bndlib_version = "1.43.0" dependencies { shadow library.java.vendored_guava_20_0 compileOnly "biz.aQute:bndlib:$bndlib_version" - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow "org.apache.tika:tika-core:$tika_version" - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testCompile "org.apache.tika:tika-parsers:$tika_version" testCompileOnly "biz.aQute:bndlib:$bndlib_version" - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/io/xml/build.gradle b/sdks/java/io/xml/build.gradle index ffc6340eef20a..c2bada0dd3807 100644 --- a/sdks/java/io/xml/build.gradle +++ b/sdks/java/io/xml/build.gradle @@ -24,14 +24,14 @@ ext.summary = "IO to read and write XML files." dependencies { shadow library.java.vendored_guava_20_0 - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") shadow library.java.stax2_api shadow library.java.woodstox_core_asl shadowTest library.java.jaxb_api - testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest") + testCompile project(path: ":sdks:java:core", configuration: "shadowTest") testCompile library.java.junit testCompile library.java.hamcrest_core testCompile library.java.hamcrest_library testRuntimeOnly library.java.slf4j_jdk14 - testRuntimeOnly project(path: ":beam-runners-direct-java") + testRuntimeOnly project(path: ":runners:direct-java") } diff --git a/sdks/java/javadoc/build.gradle b/sdks/java/javadoc/build.gradle index d32cc44bc48bf..23199caf6dc43 100644 --- a/sdks/java/javadoc/build.gradle +++ b/sdks/java/javadoc/build.gradle @@ -19,7 +19,7 @@ /* * Aggregate Javadoc is not published to Maven Central. Each Java module that is published * to Maven Cental packages its own Javadoc. To generate aggregated javadocs, run: - * ./gradlew :beam-sdks-java-javadoc:aggregateJavadoc + * ./gradlew :sdks:java:javadoc:aggregateJavadoc * Generated files will be located under beam/sdks/java/javadoc/build/docs/javadoc and are * used as part of the beam-site source tree. */ @@ -28,19 +28,19 @@ applyJavaNature() description = "Apache Beam :: SDKs :: Java :: Aggregated Javadoc" for (p in rootProject.subprojects) { - if (!p.name.equals(project.name) && !p.name.equals('beam-sdks-java-bom')) { - evaluationDependsOn(':' + p.name) + if (!p.path.equals(project.path) && !p.path.equals(':sdks:java:bom')) { + evaluationDependsOn(p.path) } } ext.getExportedJavadocProjects = { def exportedJavadocProjects = new ArrayList<>(); for (p in rootProject.subprojects) { - if (!p.name.equals(project.name) && !p.name.equals('beam-sdks-java-bom')) { - def subproject = project(':' + p.name) + if (!p.path.equals(project.path) && !p.path.equals(':sdks:java:bom')) { + def subproject = p // project(':' + p.name) if (subproject.ext.properties.containsKey('exportJavadoc') && subproject.ext.properties.exportJavadoc) { - exportedJavadocProjects.add(':' + p.name) + exportedJavadocProjects.add(p.path) } } } diff --git a/sdks/java/maven-archetypes/examples/build.gradle b/sdks/java/maven-archetypes/examples/build.gradle index bfec83712300b..2fe79a5a8f6b8 100644 --- a/sdks/java/maven-archetypes/examples/build.gradle +++ b/sdks/java/maven-archetypes/examples/build.gradle @@ -52,12 +52,12 @@ processResources { * the following projects are evaluated before we evaluate this project. This is because * we are attempting to reference the "sourceSets.{main|test}.allSource" directly. */ -evaluationDependsOn(':beam-examples-java') +evaluationDependsOn(':examples:java') task generateSources(type: Exec) { inputs.file './generate-sources.sh' - inputs.files project(':beam-examples-java').sourceSets.main.allSource - inputs.files project(':beam-examples-java').sourceSets.test.allSource + inputs.files project(':examples:java').sourceSets.main.allSource + inputs.files project(':examples:java').sourceSets.test.allSource outputs.dir('src/main/resources/archetype-resources/src') commandLine './generate-sources.sh' } @@ -69,5 +69,5 @@ sourceSets { } dependencies { - shadow project(path: ":beam-examples-java", configuration: "shadow") + shadow project(path: ":examples:java", configuration: "shadow") } diff --git a/sdks/java/maven-archetypes/starter/build.gradle b/sdks/java/maven-archetypes/starter/build.gradle index f516088db2b81..38eda091076fa 100644 --- a/sdks/java/maven-archetypes/starter/build.gradle +++ b/sdks/java/maven-archetypes/starter/build.gradle @@ -33,6 +33,6 @@ processResources { } dependencies { - shadow project(path: ":beam-runners-direct-java", configuration: "shadow") - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") + shadow project(path: ":runners:direct-java", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") } diff --git a/sdks/java/testing/load-tests/build.gradle b/sdks/java/testing/load-tests/build.gradle index 0304f294dae5c..403e83863c69e 100644 --- a/sdks/java/testing/load-tests/build.gradle +++ b/sdks/java/testing/load-tests/build.gradle @@ -17,6 +17,7 @@ */ plugins { id 'org.apache.beam.module' } +archivesBaseName = 'beam-sdks-java-load-tests' applyJavaNature(exportJavadoc: false) description = "Apache Beam :: SDKs :: Java :: Load Tests" @@ -33,10 +34,10 @@ def loadTestArgsProperty = "loadTest.args" def runnerProperty = "runner" def runnerDependency = (project.hasProperty(runnerProperty) ? project.getProperty(runnerProperty) - : ":beam-runners-direct-java") + : ":runners:direct-java") -def shouldProvideSpark = ":beam-runners-spark".equals(runnerDependency) -def isDataflowRunner = ":beam-runners-google-cloud-dataflow-java".equals(runnerDependency) +def shouldProvideSpark = ":runners:spark".equals(runnerDependency) +def isDataflowRunner = ":runners:google-cloud-dataflow-java".equals(runnerDependency) if (isDataflowRunner) { /* @@ -44,7 +45,7 @@ if (isDataflowRunner) { * the following projects are evaluated before we evaluate this project. This is because * we are attempting to reference a property from the project directly. */ - evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker") + evaluationDependsOn(":runners:google-cloud-dataflow-java:worker:legacy-worker") } configurations { @@ -57,20 +58,20 @@ configurations { dependencies { shadow library.java.kafka_clients - shadow project(path: ":beam-sdks-java-core", configuration: "shadow") - shadow project(path: ":beam-runners-direct-java", configuration: "shadow") - shadow project(path: ":beam-sdks-java-io-synthetic", configuration: "shadow") - shadow project(path: ":beam-sdks-java-test-utils", configuration: "shadow") - shadow project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") - shadow project(path: ":beam-sdks-java-io-kafka", configuration: "shadow") - shadow project(path: ":beam-sdks-java-io-kinesis", configuration: "shadow") + shadow project(path: ":sdks:java:core", configuration: "shadow") + shadow project(path: ":runners:direct-java", configuration: "shadow") + shadow project(path: ":sdks:java:io:synthetic", configuration: "shadow") + shadow project(path: ":sdks:java:testing:test-utils", configuration: "shadow") + shadow project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow") + shadow project(path: ":sdks:java:io:kafka", configuration: "shadow") + shadow project(path: ":sdks:java:io:kinesis", configuration: "shadow") gradleRun project(path: project.path, configuration: "shadow") gradleRun project(path: runnerDependency, configuration: "shadow") // The Spark runner requires the user to provide a Spark dependency. For self-contained // runs with the Spark runner, we can provide such a dependency. This is deliberately phrased - // to not hardcode any runner other than :beam-runners-direct-java + // to not hardcode any runner other than :runners:direct-java if (shouldProvideSpark) { gradleRun library.java.spark_streaming gradleRun library.java.spark_core, { @@ -90,9 +91,9 @@ task run(type: JavaExec) { def loadTestArgs = project.findProperty(loadTestArgsProperty) ?: "" if (isDataflowRunner) { - dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" + dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath // Provide job with a customizable worker jar. // With legacy worker jar, containerImage is set to empty (i.e. to use the internal build). // More context and discussions can be found in PR#6694. diff --git a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CoGroupByKeyLoadTest.java b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CoGroupByKeyLoadTest.java index c78af22e42de7..ec6b96bba921a 100644 --- a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CoGroupByKeyLoadTest.java +++ b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CoGroupByKeyLoadTest.java @@ -51,7 +51,7 @@ *

To run it manually, use the following command: * * 

- *    ./gradlew :beam-sdks-java-load-tests:run -PloadTest.args='
+ *    ./gradlew :sdks:java:testing:load-tests:run -PloadTest.args='
  *      --iterations=1
  *      --sourceOptions={"numRecords":1000,...}
  *      --coSourceOptions={"numRecords":1000,...}
diff --git a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CombineLoadTest.java b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CombineLoadTest.java
index b625367af8172..b3fe2339a5304 100644
--- a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CombineLoadTest.java
+++ b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/CombineLoadTest.java
@@ -54,7 +54,7 @@
  * 
To run it manually, use the following command:
  *
  * 
- *    ./gradlew :beam-sdks-java-load-tests:run -PloadTest.args='
+ *    ./gradlew :sdks:java:testing:load-tests:run -PloadTest.args='
  *      --fanout=1
  *      --perKeyCombinerType=TOP_LARGEST
  *      --topCount=10
diff --git a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/GroupByKeyLoadTest.java b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/GroupByKeyLoadTest.java
index 611896750be1a..4a569f79af8d8 100644
--- a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/GroupByKeyLoadTest.java
+++ b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/GroupByKeyLoadTest.java
@@ -46,7 +46,7 @@
  * 
To run it manually, use the following command:
  *
  * 
- *    ./gradlew :beam-sdks-java-load-tests:run -PloadTest.args='
+ *    ./gradlew :sdks:java:testing:load-tests:run -PloadTest.args='
  *      --fanout=1
  *      --iterations=1
  *      --sourceOptions={"numRecords":1000,...}
diff --git a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/ParDoLoadTest.java b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/ParDoLoadTest.java
index 61884ededf47e..6b0128080d8c9 100644
--- a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/ParDoLoadTest.java
+++ b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/ParDoLoadTest.java
@@ -38,7 +38,7 @@
  * 
To run it manually, use the following command:
  *
  * 
- *    ./gradlew :beam-sdks-java-load-tests:run -PloadTest.args='
+ *    ./gradlew :sdks:java:testing:load-tests:run -PloadTest.args='
  *      --numberOfCounterOperations=1
  *      --sourceOptions={"numRecords":1000,...}
  *      --stepOptions={"outputRecordsPerInputRecord":2...}'
diff --git a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java
index e35054fcd23f7..8c7207647facf 100644
--- a/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java
+++ b/sdks/java/testing/load-tests/src/main/java/org/apache/beam/sdk/loadtests/SyntheticDataPublisher.java
@@ -55,7 +55,7 @@
  * 
To run it manually, use the following command:
  *
  * 
- *  ./gradlew :beam-sdks-java-load-tests:run -PloadTest.args='
+ *  ./gradlew :sdks:java:testing:load-tests:run -PloadTest.args='
  *    --pubSubTopic=TOPIC_NAME
  *    --kafkaBootstrapServerAddress=SERVER_ADDRESS
  *    --kafkaTopic=KAFKA_TOPIC_NAME
diff --git a/sdks/java/testing/nexmark/build.gradle b/sdks/java/testing/nexmark/build.gradle
index a1db48ad18dcf..53446319ec2f6 100644
--- a/sdks/java/testing/nexmark/build.gradle
+++ b/sdks/java/testing/nexmark/build.gradle
@@ -17,6 +17,7 @@
  */
 
 plugins { id 'org.apache.beam.module' }
+archivesBaseName = 'beam-sdks-java-nexmark'
 applyJavaNature(testShadowJar: true, exportJavadoc: false)
 
 description = "Apache Beam :: SDKs :: Java :: Nexmark"
@@ -28,9 +29,9 @@ def nexmarkArgsProperty = "nexmark.args"
 // When running via Gradle, this property sets the runner dependency
 def nexmarkRunnerProperty = "nexmark.runner"
 def nexmarkRunnerDependency = project.findProperty(nexmarkRunnerProperty)
-        ?: ":beam-runners-direct-java"
-def shouldProvideSpark = ":beam-runners-spark".equals(nexmarkRunnerDependency)
-def isDataflowRunner = ":beam-runners-google-cloud-dataflow-java".equals(nexmarkRunnerDependency)
+        ?: ":runners:direct-java"
+def shouldProvideSpark = ":runners:spark".equals(nexmarkRunnerDependency)
+def isDataflowRunner = ":runners:google-cloud-dataflow-java".equals(nexmarkRunnerDependency)
 
 if (isDataflowRunner) {
   /*
@@ -38,7 +39,7 @@ if (isDataflowRunner) {
    * the following projects are evaluated before we evaluate this project. This is because
    * we are attempting to reference a property from the project directly.
    */
-  evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker")
+  evaluationDependsOn(":runners:google-cloud-dataflow-java:worker:legacy-worker")
 }
 
 configurations {
@@ -50,12 +51,12 @@ configurations {
 
 dependencies {
   shadow library.java.vendored_guava_20_0
-  shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
-  shadow project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow")
-  shadow project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadow")
-  shadow project(path: ":beam-sdks-java-extensions-sql", configuration: "shadow")
-  shadow project(path: ":beam-sdks-java-io-kafka", configuration: "shadow")
-  shadow project(path: ":beam-sdks-java-test-utils", configuration: "shadow")
+  shadow project(path: ":sdks:java:core", configuration: "shadow")
+  shadow project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow")
+  shadow project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadow")
+  shadow project(path: ":sdks:java:extensions:sql", configuration: "shadow")
+  shadow project(path: ":sdks:java:io:kafka", configuration: "shadow")
+  shadow project(path: ":sdks:java:testing:test-utils", configuration: "shadow")
   shadow library.java.google_api_services_bigquery
   shadow library.java.jackson_core
   shadow library.java.jackson_annotations
@@ -65,13 +66,13 @@ dependencies {
   shadow library.java.slf4j_api
   shadow library.java.commons_lang3
   shadow library.java.kafka_clients
-  shadow project(path: ":beam-runners-direct-java", configuration: "shadow")
+  shadow project(path: ":runners:direct-java", configuration: "shadow")
   provided library.java.junit
   provided library.java.hamcrest_core
-  shadow project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow")
+  shadow project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadow")
   shadowTestRuntimeClasspath library.java.slf4j_jdk14
-  shadowTest project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadowTest")
-  shadowTest project(path: ":beam-sdks-java-test-utils", configuration: "shadowTest")
+  shadowTest project(path: ":sdks:java:io:google-cloud-platform", configuration: "shadowTest")
+  shadowTest project(path: ":sdks:java:testing:test-utils", configuration: "shadowTest")
   testCompile library.java.hamcrest_core
   testCompile library.java.hamcrest_library
 
@@ -80,7 +81,7 @@ dependencies {
 
   // The Spark runner requires the user to provide a Spark dependency. For self-contained
   // runs with the Spark runner, we can provide such a dependency. This is deliberately phrased
-  // to not hardcode any runner other than :beam-runners-direct-java
+  // to not hardcode any runner other than :runners:direct-java
   if (shouldProvideSpark) {
     gradleRun library.java.spark_streaming
     gradleRun library.java.spark_core, {
@@ -100,8 +101,8 @@ if (shouldProvideSpark) {
 //
 // Parameters:
 //   -Pnexmark.runner
-//       Specify a runner subproject, such as ":beam-runners-spark" or ":beam-runners-flink"
-//       Defaults to ":beam-runners-direct-java"
+//       Specify a runner subproject, such as ":runners:spark" or ":runners:flink:1.5"
+//       Defaults to ":runners:direct-java"
 //
 //   -Pnexmark.args
 //       Specify the command line for invoking org.apache.beam.sdk.nexmark.Main
@@ -109,9 +110,9 @@ task run(type: JavaExec) {
   def nexmarkArgsStr =  project.findProperty(nexmarkArgsProperty) ?: ""
 
   if (isDataflowRunner) {
-    dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
+    dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar"
 
-    def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
+    def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath
     // Provide job with a customizable worker jar.
     // With legacy worker jar, containerImage is set to empty (i.e. to use the internal build).
     // More context and discussions can be found in PR#6694.
diff --git a/sdks/java/testing/test-utils/build.gradle b/sdks/java/testing/test-utils/build.gradle
index 4f3d6de56a25c..a073b5d0d97c8 100644
--- a/sdks/java/testing/test-utils/build.gradle
+++ b/sdks/java/testing/test-utils/build.gradle
@@ -17,19 +17,20 @@
  */
 
 plugins { id 'org.apache.beam.module' }
+archivesBaseName = 'beam-sdks-java-test-utils'
 applyJavaNature(exportJavadoc: false)
 
 description = "Apache Beam :: SDKs :: Java :: Test Utils"
 
 dependencies {
-  shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
+  shadow project(path: ":sdks:java:core", configuration: "shadow")
   shadow library.java.vendored_guava_20_0
   shadow library.java.google_cloud_bigquery
-  shadow project(path: ":beam-sdks-java-extensions-google-cloud-platform-core", configuration: "shadow")
+  shadow project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "shadow")
 
   shadowTest library.java.junit
   shadowTest library.java.mockito_core
   shadowTest library.java.hamcrest_core
   shadowTest library.java.hamcrest_library
-  testRuntimeOnly project(path: ":beam-runners-direct-java")
+  testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadowTest")
 }
diff --git a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
index 8312f07316775..26c865566a546 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
@@ -31,7 +31,7 @@
 permissions for the key specified in --kms_key_name.
 
 To run these tests manually:
-  ./gradlew beam-sdks-python:integrationTest \
+  ./gradlew :sdks:python:integrationTest \
     -Dtests=apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest \
     -DkmsKeyName=KMS_KEY_NAME
 """
diff --git a/sdks/python/apache_beam/testing/load_tests/co_group_by_key_test.py b/sdks/python/apache_beam/testing/load_tests/co_group_by_key_test.py
index 4a6f30b7e6fad..6c050c316c270 100644
--- a/sdks/python/apache_beam/testing/load_tests/co_group_by_key_test.py
+++ b/sdks/python/apache_beam/testing/load_tests/co_group_by_key_test.py
@@ -76,7 +76,7 @@
     --runner=DirectRunner' \
 -PloadTest.mainClass=
 apache_beam.testing.load_tests.co_group_by_key_test \
--Prunner=DirectRunner :beam-sdks-python-load-tests:run
+-Prunner=DirectRunner :sdks:python:apache_beam:testing:load-tests:run
 
 To run test on other runner (ex. Dataflow):
 
@@ -133,7 +133,7 @@
     --runner=TestDataflowRunner' \
 -PloadTest.mainClass=
 apache_beam.testing.load_tests.co_group_by_key_test \
--Prunner=TestDataflowRunner :beam-sdks-python-load-tests:run
+-Prunner=TestDataflowRunner :sdks:python:apache_beam:testing:load-tests:run
 """
 
 from __future__ import absolute_import
diff --git a/sdks/python/apache_beam/testing/load_tests/combine_test.py b/sdks/python/apache_beam/testing/load_tests/combine_test.py
index 3573fa862a5b3..371f52e69d239 100644
--- a/sdks/python/apache_beam/testing/load_tests/combine_test.py
+++ b/sdks/python/apache_beam/testing/load_tests/combine_test.py
@@ -64,7 +64,7 @@
     --runner=DirectRunner
     --fanout=1' \
 -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \
--Prunner=DirectRunner :beam-sdks-python-load-tests:run
+-Prunner=DirectRunner :sdks:python:apache_beam:testing:load-tests:run
 
 To run test on other runner (ex. Dataflow):
 
@@ -109,7 +109,7 @@
 -PloadTest.mainClass=
 apache_beam.testing.load_tests.combine_test \
 -Prunner=
-TestDataflowRunner :beam-sdks-python-load-tests:run
+TestDataflowRunner :sdks:python:apache_beam:testing:load-tests:run
 """
 
 from __future__ import absolute_import
diff --git a/sdks/python/apache_beam/testing/load_tests/group_by_key_test.py b/sdks/python/apache_beam/testing/load_tests/group_by_key_test.py
index 1d579bce54fd6..d19e7f703a32a 100644
--- a/sdks/python/apache_beam/testing/load_tests/group_by_key_test.py
+++ b/sdks/python/apache_beam/testing/load_tests/group_by_key_test.py
@@ -69,7 +69,7 @@
     --runner=DirectRunner' \
 -PloadTest.mainClass=
 apache_beam.testing.load_tests.group_by_key_test \
--Prunner=DirectRunner :beam-sdks-python-load-tests:run
+-Prunner=DirectRunner :sdks:python:apache_beam:testing:load-tests:run
 
 To run test on other runner (ex. Dataflow):
 
@@ -115,7 +115,7 @@
     --runner=TestDataflowRunner' \
 -PloadTest.mainClass=
 apache_beam.testing.load_tests.group_by_key_test \
--Prunner=TestDataflowRunner :beam-sdks-python-load-tests:run
+-Prunner=TestDataflowRunner :sdks:python:apache_beam:testing:load-tests:run
 """
 
 from __future__ import absolute_import
diff --git a/sdks/python/apache_beam/testing/load_tests/pardo_test.py b/sdks/python/apache_beam/testing/load_tests/pardo_test.py
index 908431ef2aca2..873be2ee8017a 100644
--- a/sdks/python/apache_beam/testing/load_tests/pardo_test.py
+++ b/sdks/python/apache_beam/testing/load_tests/pardo_test.py
@@ -66,7 +66,7 @@
       "force_initial_num_bundles": 1}\'
     --runner=DirectRunner' \
 -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \
--Prunner=DirectRunner :beam-sdks-python-load-tests:run
+-Prunner=DirectRunner :sdks:python:apache_beam:testing:load-tests:run
 
 
 To run test on other runner (ex. Dataflow):
@@ -110,7 +110,7 @@
       "force_initial_num_bundles": 1}\'
     --runner=TestDataflowRunner' \
 -PloadTest.mainClass=apache_beam.testing.load_tests.pardo_test \
--Prunner=TestDataflowRunner :beam-sdks-python-load-tests:run
+-Prunner=TestDataflowRunner :sdks:python:apache_beam:testing:load-tests:run
 """
 
 from __future__ import absolute_import
diff --git a/sdks/python/apache_beam/testing/load_tests/sideinput_test.py b/sdks/python/apache_beam/testing/load_tests/sideinput_test.py
index c20f48ae5a803..4143f8322c425 100644
--- a/sdks/python/apache_beam/testing/load_tests/sideinput_test.py
+++ b/sdks/python/apache_beam/testing/load_tests/sideinput_test.py
@@ -65,7 +65,7 @@
     --runner=DirectRunner' \
 -PloadTest.mainClass=
 apache_beam.testing.load_tests.sideinput_test \
--Prunner=DirectRunner :beam-sdks-python-load-tests:run
+-Prunner=DirectRunner :sdks:python:apache_beam:testing:load-tests:run
 
 To run test on other runner (ex. Dataflow):
 
@@ -109,7 +109,7 @@
     --runner=TestDataflowRunner' \
 -PloadTest.mainClass=
 apache_beam.testing.load_tests.sideinput_test:SideInputTest.testSideInput \
--Prunner=TestDataflowRunner :beam-sdks-python-load-tests:run
+-Prunner=TestDataflowRunner :sdks:python:apache_beam:testing:load-tests:run
 """
 
 from __future__ import absolute_import
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 49feebf18f23c..7bd39844a7a93 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -78,8 +78,8 @@ task preCommitPy2() {
 }
 
 task portablePreCommit() {
-  dependsOn ':beam-runners-flink_2.11-job-server-container:docker'
-  dependsOn ':beam-sdks-python-container:docker'
+  dependsOn ':runners:flink:1.5:job-server-container:docker'
+  dependsOn ':sdks:python:container:docker'
   dependsOn portableWordCountTask('portableWordCountBatch', false)
   dependsOn portableWordCountTask('portableWordCountStreaming', true)
 }
@@ -142,22 +142,22 @@ task directRunnerIT(dependsOn: 'installGcpTest') {
 // 2. Either a) or b)
 //  a) If you want the Job Server to run in a Docker container:
 //
-//    ./gradlew :beam-runners-flink_2.11-job-server-container:docker
+//    ./gradlew :runners:flink:1.5:job-server-container:docker
 //
 //  b) Otherwise, start a local JobService, for example, the Portable Flink runner
 //    (in a separate shell since it continues to run):
 //
-//    ./gradlew :beam-runners-flink_2.11-job-server:runShadow
+//    ./gradlew :runners:flink:1.5:job-server:runShadow
 //
 // Then you can run this example:
 //
 //  Docker (2a):
 //
-//    ./gradlew :beam-sdks-python:portableWordCount
+//    ./gradlew :sdks:python:portableWordCount
 //
 //  Local JobService (2b):
 //
-//    ./gradlew :beam-sdks-python:portableWordCount -PjobEndpoint=localhost:8099
+//    ./gradlew :sdks:python:portableWordCount -PjobEndpoint=localhost:8099
 //
 task portableWordCount {
   dependsOn portableWordCountTask('portableWordCountExample', project.hasProperty("streaming"))
@@ -166,7 +166,7 @@ task portableWordCount {
 def portableWordCountTask(name, streaming) {
   tasks.create(name) {
     dependsOn = ['installGcpTest']
-    mustRunAfter = [':beam-runners-flink_2.11-job-server-container:docker', ':beam-sdks-python-container:docker']
+    mustRunAfter = [':runners:flink:1.5:job-server-container:docker', ':sdks:python:container:docker']
     doLast {
       // TODO: Figure out GCS credentials and use real GCS input and output.
       def options = [
@@ -201,9 +201,9 @@ def portableWordCountTask(name, streaming) {
 
 // Run PostCommit integration tests on default runner (TestDataflowRunner)
 task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) {
-  dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
 
-  def dataflowWorkerJar = project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
 
   doLast {
     def testOpts = basicTestOpts + ["--attr=IT"]
@@ -217,9 +217,9 @@ task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) {
 }
 
 task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) {
-  dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
 
-  def dataflowWorkerJar = project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
 
   doLast {
     def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
@@ -233,9 +233,9 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) {
 }
 
 task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) {
-  dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
 
-  def dataflowWorkerJar = project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
 
   doLast {
     // TODO(BEAM-3544,BEAM-5025): Disable tests with 'sickbay-streaming' tag.
@@ -304,22 +304,22 @@ def flinkCompatibilityMatrix = {
   def config = it ? it as CompatibilityMatrixConfig : new CompatibilityMatrixConfig()
   def workerType = config.workerType.name()
   def streaming = config.streaming
-  def environment_config = config.workerType == CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS ? "--environment_config='{\"command\": \"${project(":beam-sdks-python:").buildDir.absolutePath}/sdk_worker.sh\"}'" : ""
+  def environment_config = config.workerType == CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS ? "--environment_config='{\"command\": \"${project(":sdks:python").buildDir.absolutePath}/sdk_worker.sh\"}'" : ""
   def name = "flinkCompatibilityMatrix${streaming ? 'Streaming' : 'Batch'}${config.preOptimize ? 'PreOptimize' : ''}${workerType}"
   def extra_experiments = []
   if (config.preOptimize)
     extra_experiments.add('pre_optimize=all')
   tasks.create(name: name) {
     dependsOn 'setupVirtualenv'
-    dependsOn ':beam-runners-flink_2.11-job-server:shadowJar'
+    dependsOn ':runners:flink:1.5:job-server:shadowJar'
     if (workerType.toLowerCase() == 'docker')
-      dependsOn ':beam-sdks-python-container:docker'
+      dependsOn ':sdks:python:container:docker'
     else if (workerType.toLowerCase() == 'process')
       dependsOn 'createProcessWorker'
     doLast {
       exec {
         executable 'sh'
-        args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e .[test] && python -m apache_beam.runners.portability.flink_runner_test --flink_job_server_jar=${project(":beam-runners-flink_2.11-job-server:").shadowJar.archivePath} --environment_type=${workerType} ${environment_config} ${streaming ? '--streaming' : ''} ${extra_experiments ? '--extra_experiments=' + extra_experiments.join(',') : ''}"
+        args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e .[test] && python -m apache_beam.runners.portability.flink_runner_test --flink_job_server_jar=${project(":runners:flink:1.5:job-server:").shadowJar.archivePath} --environment_type=${workerType} ${environment_config} ${streaming ? '--streaming' : ''} ${extra_experiments ? '--extra_experiments=' + extra_experiments.join(',') : ''}"
       }
     }
   }
@@ -349,12 +349,12 @@ task flinkValidatesRunner() {
 // the SDK environment.
 task javaReferenceRunnerValidatesRunner() {
   dependsOn 'setupVirtualenv'
-  dependsOn ':beam-runners-reference-job-server:shadowJar'
-  dependsOn ':beam-sdks-python-container:docker'
+  dependsOn ':runners:reference:job-server:shadowJar'
+  dependsOn ':sdks:python:container:docker'
   doLast {
     exec {
       executable 'sh'
-      args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e .[test] && python -m apache_beam.runners.portability.java_reference_runner_test --job_server_jar=${project(":beam-runners-reference-job-server:").shadowJar.archivePath} --environment_type=DOCKER"
+      args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e .[test] && python -m apache_beam.runners.portability.java_reference_runner_test --job_server_jar=${project(":runners:reference:job-server:").shadowJar.archivePath} --environment_type=DOCKER"
     }
   }
 }
@@ -398,13 +398,13 @@ task buildSnapshot() {
 }
 
 project.task('createProcessWorker') {
-  dependsOn ':beam-sdks-python-container:build'
+  dependsOn ':sdks:python:container:build'
   dependsOn 'setupVirtualenv'
   def sdkWorkerFile = file("${project.buildDir}/sdk_worker.sh")
   def osType = 'linux'
   if (Os.isFamily(Os.FAMILY_MAC))
     osType = 'darwin'
-  def workerScript = "${project(":beam-sdks-python-container:").buildDir.absolutePath}/target/launcher/${osType}_amd64/boot"
+  def workerScript = "${project(":sdks:python:container:").buildDir.absolutePath}/target/launcher/${osType}_amd64/boot"
   def sdkWorkerFileCode = "sh -c \"pip=`which pip` . ${project.ext.envdir}/bin/activate && ${workerScript} \$* \""
   outputs.file sdkWorkerFile
   doLast {
@@ -420,13 +420,13 @@ project.task('createProcessWorker') {
 
 project.task('crossLanguagePythonJavaFlink') {
   dependsOn 'setupVirtualenv'
-  dependsOn ':beam-runners-flink_2.11-job-server-container:docker'
-  dependsOn ':beam-sdks-python-container:docker'
-  dependsOn ':beam-sdks-java-container:docker'
-  dependsOn ':beam-runners-core-construction-java:buildTestExpansionServiceJar'
+  dependsOn ':runners:flink:1.5:job-server-container:docker'
+  dependsOn ':sdks:python:container:docker'
+  dependsOn ':sdks:java:container:docker'
+  dependsOn ':runners:core-construction-java:buildTestExpansionServiceJar'
 
   doLast {
-    def testServiceExpansionJar = project(":beam-runners-core-construction-java:").buildTestExpansionServiceJar.archivePath
+    def testServiceExpansionJar = project(":runners:core-construction-java:").buildTestExpansionServiceJar.archivePath
     def options = [
             "--runner=PortableRunner",
             "--experiments=worker_threads=100",
@@ -445,13 +445,13 @@ project.task('crossLanguagePythonJavaFlink') {
 
 project.task('crossLanguagePortableWordCount') {
   dependsOn 'setupVirtualenv'
-  dependsOn ':beam-runners-flink_2.11-job-server-container:docker'
-  dependsOn ':beam-sdks-python-container:docker'
-  dependsOn ':beam-sdks-java-container:docker'
-  dependsOn ':beam-runners-core-construction-java:buildTestExpansionServiceJar'
+  dependsOn ':runners:flink:1.5:job-server-container:docker'
+  dependsOn ':sdks:python:container:docker'
+  dependsOn ':sdks:java:container:docker'
+  dependsOn ':runners:core-construction-java:buildTestExpansionServiceJar'
 
   doLast {
-    def testServiceExpansionJar = project(":beam-runners-core-construction-java:").buildTestExpansionServiceJar.archivePath
+    def testServiceExpansionJar = project(":runners:core-construction-java:").buildTestExpansionServiceJar.archivePath
     def options = [
             "--input=/etc/profile",
             "--output=/tmp/py-wordcount-portable",
diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle
index 4dee86c9d0275..a6650e9889750 100644
--- a/sdks/python/container/build.gradle
+++ b/sdks/python/container/build.gradle
@@ -25,7 +25,7 @@ description = "Apache Beam :: SDKs :: Python :: Container"
 // Figure out why the golang plugin does not add a build dependency between projects.
 // Without the line below, we get spurious errors about not being able to resolve
 // "./github.com/apache/beam/sdks/go"
-resolveBuildDependencies.dependsOn ":beam-sdks-go:goBuild"
+resolveBuildDependencies.dependsOn ":sdks:go:goBuild"
 
 configurations {
   sdkSourceTarball
@@ -36,10 +36,10 @@ dependencies {
   golang {
     // TODO(herohde): use "./" prefix to prevent gogradle use base github path, for now.
     // TODO(herohde): get the pkg subdirectory only, if possible. We spend mins pulling cmd/beamctl deps.
-    build name: './github.com/apache/beam/sdks/go', dir: project(':beam-sdks-go').projectDir
-    test name: './github.com/apache/beam/sdks/go', dir: project(':beam-sdks-go').projectDir
+    build name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir
+    test name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir
   }
-  sdkSourceTarball project(path: ":beam-sdks-python", configuration: "distConfig")
+  sdkSourceTarball project(path: ":sdks:python", configuration: "distConfig")
 }
 
 task copyDockerfileDependencies(type: Copy, dependsOn: goBuild) {
diff --git a/sdks/python/container/py3/build.gradle b/sdks/python/container/py3/build.gradle
index b577b7756b59d..05710bc35866e 100644
--- a/sdks/python/container/py3/build.gradle
+++ b/sdks/python/container/py3/build.gradle
@@ -30,8 +30,8 @@ configurations {
 }
 
 dependencies {
-  sdkSourceTarball project(path: ":beam-sdks-python", configuration: "distConfig")
-  sdkHarnessLauncher project(path: ":beam-sdks-python-container", configuration: "sdkHarnessLauncher")
+  sdkSourceTarball project(path: ":sdks:python", configuration: "distConfig")
+  sdkHarnessLauncher project(path: ":sdks:python:container", configuration: "sdkHarnessLauncher")
 }
 
 task copyDockerfileDependencies(type: Copy) {
diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh
index 47f5b8d9c8b4f..639f4dcd4b0dc 100755
--- a/sdks/python/container/run_validatescontainer.sh
+++ b/sdks/python/container/run_validatescontainer.sh
@@ -48,12 +48,12 @@ PROJECT=${PROJECT:-apache-beam-testing}
 # Other variables branched by Python version.
 if [[ $1 == "python2" ]]; then
   IMAGE_NAME="python"       # Use this to create CONTAINER_IMAGE variable.
-  CONTAINER_PROJECT="beam-sdks-python-container"  # Use this to build container by Gradle.
+  CONTAINER_PROJECT="sdks:python:container"  # Use this to build container by Gradle.
   GRADLE_PY3_FLAG=""        # Use this in Gradle command.
   PY_INTERPRETER="python"   # Use this in virtualenv command.
 elif [[ $1 == "python3" ]]; then
   IMAGE_NAME="python3"          # Use this to create CONTAINER_IMAGE variable.
-  CONTAINER_PROJECT="beam-sdks-python-container-py3"  # Use this to build container by Gradle.
+  CONTAINER_PROJECT="sdks:python:container:py3"  # Use this to build container by Gradle.
   GRADLE_PY3_FLAG="-Ppython3"   # Use this in Gradle command.
   PY_INTERPRETER="python3.5"    # Use this in virtualenv command.
 else
diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh
index ba72a14be3178..6e7b3ca2d20ed 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -21,7 +21,7 @@
 # This script is useful to run single or a set of Python integration tests
 # manually or through Gradle. Note, this script doesn't setup python
 # environment which is required before running tests. Use Gradle task
-# `beam-sdks-python:integrationTests` to do both together.
+# `:sdks:python:integrationTests` to do both together.
 #
 # In order to run test with customer options, use following commandline flags:
 #
diff --git a/sdks/python/scripts/run_snapshot_publish.sh b/sdks/python/scripts/run_snapshot_publish.sh
index 302dcbf0d268f..5cdde16f53a22 100755
--- a/sdks/python/scripts/run_snapshot_publish.sh
+++ b/sdks/python/scripts/run_snapshot_publish.sh
@@ -26,7 +26,7 @@ SNAPSHOT="apache-beam-$VERSION-$time.zip"
 DEP_SNAPSHOT_ROOT="$BUCKET/dependency_requirements_snapshot"
 DEP_SNAPSHOT_FILE_NAME="beam-py-requirements-$time.txt"
 
-# Snapshots are built by Gradle task :beam-sdks-python:depSnapshot
+# Snapshots are built by Gradle task :sdks:python:depSnapshot
 # and located under Gradle build directory.
 cd $WORKSPACE/src/sdks/python/build
 
diff --git a/sdks/python/test-suites/dataflow/py35/build.gradle b/sdks/python/test-suites/dataflow/py35/build.gradle
index 7d31cb36f0085..c5b21d23639f2 100644
--- a/sdks/python/test-suites/dataflow/py35/build.gradle
+++ b/sdks/python/test-suites/dataflow/py35/build.gradle
@@ -33,9 +33,9 @@ def basicTestOpts = [
 ]
 
 task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) {
-  dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
 
-  def dataflowWorkerJar = project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
 
   doLast {
     def testOpts = basicTestOpts + ["--attr=IT"]
@@ -53,9 +53,9 @@ task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) {
 }
 
 task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) {
-  dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
 
-  def dataflowWorkerJar = project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
 
   doLast {
     def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
diff --git a/sdks/python/test-suites/dataflow/py36/build.gradle b/sdks/python/test-suites/dataflow/py36/build.gradle
index b685d05612903..4900d1cb6fd06 100644
--- a/sdks/python/test-suites/dataflow/py36/build.gradle
+++ b/sdks/python/test-suites/dataflow/py36/build.gradle
@@ -32,9 +32,9 @@ def basicTestOpts = [
 ]
 
 task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) {
-  dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
 
-  def dataflowWorkerJar = project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
 
   doLast {
     def testOpts = basicTestOpts + ["--attr=IT"]
@@ -52,9 +52,9 @@ task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) {
 }
 
 task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) {
-  dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
 
-  def dataflowWorkerJar = project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
 
   doLast {
     def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
diff --git a/settings.gradle b/settings.gradle
index a27cf3e73e0e2..fd1201beaaab0 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -20,239 +20,127 @@ rootProject.name = "beam"
 
 include ":release"
 
-include "beam-examples-kotlin"
-project(":beam-examples-kotlin").dir = file("examples/kotlin")
-include "beam-examples-java"
-project(":beam-examples-java").dir = file("examples/java")
-include "beam-model-fn-execution"
-project(":beam-model-fn-execution").dir = file("model/fn-execution")
-include "beam-model-job-management"
-project(":beam-model-job-management").dir = file("model/job-management")
-include "beam-model-pipeline"
-project(":beam-model-pipeline").dir = file("model/pipeline")
-include "beam-runners-apex"
-project(":beam-runners-apex").dir = file("runners/apex")
-include "beam-runners-core-construction-java"
-project(":beam-runners-core-construction-java").dir = file("runners/core-construction-java")
-include "beam-runners-core-java"
-project(":beam-runners-core-java").dir = file("runners/core-java")
-include "beam-runners-direct-java"
-project(":beam-runners-direct-java").dir = file("runners/direct-java")
-include "beam-runners-extensions-java-metrics"
-project(":beam-runners-extensions-java-metrics").dir = file("runners/extensions-java/metrics")
+include ":examples:java"
+include ":examples:kotlin"
+include ":model:fn-execution"
+include ":model:job-management"
+include ":model:pipeline"
+include ":runners:apex"
+include ":runners:core-construction-java"
+include ":runners:core-java"
+include ":runners:direct-java"
+include ":runners:extensions-java:metrics"
 /* Begin Flink Runner related settings */
 // Flink 1.5 (with Scala 2.11 suffix)
-include "beam-runners-flink_2.11"
-project(":beam-runners-flink_2.11").dir = file("runners/flink/1.5")
-include "beam-runners-flink_2.11-job-server"
-project(":beam-runners-flink_2.11-job-server").dir = file("runners/flink/1.5/job-server")
-include "beam-runners-flink_2.11-job-server-container"
-project(":beam-runners-flink_2.11-job-server-container").dir = file("runners/flink/1.5/job-server-container")
+include ":runners:flink:1.5"
+include ":runners:flink:1.5:job-server"
+include ":runners:flink:1.5:job-server-container"
 // Flink 1.6
-include "beam-runners-flink-1.6"
-project(":beam-runners-flink-1.6").dir = file("runners/flink/1.6")
-include "beam-runners-flink-1.6-job-server"
-project(":beam-runners-flink-1.6-job-server").dir = file("runners/flink/1.6/job-server")
-include "beam-runners-flink-1.6-job-server-container"
-project(":beam-runners-flink-1.6-job-server-container").dir = file("runners/flink/1.6/job-server-container")
+include ":runners:flink:1.6"
+include ":runners:flink:1.6:job-server"
+include ":runners:flink:1.6:job-server-container"
 // Flink 1.7
-include "beam-runners-flink-1.7"
-project(":beam-runners-flink-1.7").dir = file("runners/flink/1.7")
-include "beam-runners-flink-1.7-job-server"
-project(":beam-runners-flink-1.7-job-server").dir = file("runners/flink/1.7/job-server")
-include "beam-runners-flink-1.7-job-server-container"
-project(":beam-runners-flink-1.7-job-server-container").dir = file("runners/flink/1.7/job-server-container")
+include ":runners:flink:1.7"
+include ":runners:flink:1.7:job-server"
+include ":runners:flink:1.7:job-server-container"
 // Flink 1.8
-include "beam-runners-flink-1.8"
-project(":beam-runners-flink-1.8").dir = file("runners/flink/1.8")
-include "beam-runners-flink-1.8-job-server"
-project(":beam-runners-flink-1.8-job-server").dir = file("runners/flink/1.8/job-server")
-include "beam-runners-flink-1.8-job-server-container"
-project(":beam-runners-flink-1.8-job-server-container").dir = file("runners/flink/1.8/job-server-container")
+include ":runners:flink:1.8"
+include ":runners:flink:1.8:job-server"
+include ":runners:flink:1.8:job-server-container"
 /* End Flink Runner related settings */
-include "beam-runners-gearpump"
-project(":beam-runners-gearpump").dir = file("runners/gearpump")
-include "beam-runners-google-cloud-dataflow-java"
-project(":beam-runners-google-cloud-dataflow-java").dir = file("runners/google-cloud-dataflow-java")
-include ":beam-runners-google-cloud-dataflow-java-examples"
-project(":beam-runners-google-cloud-dataflow-java-examples").dir = file("runners/google-cloud-dataflow-java/examples")
-include ":beam-runners-google-cloud-dataflow-java-examples-streaming"
-project(":beam-runners-google-cloud-dataflow-java-examples-streaming").dir = file("runners/google-cloud-dataflow-java/examples-streaming")
-include "beam-runners-java-fn-execution"
-project(":beam-runners-java-fn-execution").dir = file("runners/java-fn-execution")
-include "beam-runners-local-java-core"
-project(":beam-runners-local-java-core").dir = file("runners/local-java")
-include "beam-runners-reference-java"
-project(":beam-runners-reference-java").dir = file("runners/reference/java")
-include "beam-runners-reference-job-server"
-project(":beam-runners-reference-job-server").dir = file("runners/reference/job-server")
-include "beam-runners-spark"
-project(":beam-runners-spark").dir = file("runners/spark")
-include "beam-runners-spark-job-server"
-project(":beam-runners-spark-job-server").dir = file("runners/spark/job-server")
-include "beam-runners-samza"
-project(":beam-runners-samza").dir = file("runners/samza")
-include "beam-runners-samza-job-server"
-project(":beam-runners-samza-job-server").dir = file("runners/samza/job-server")
-include "beam-sdks-go"
-project(":beam-sdks-go").dir = file("sdks/go")
-include "beam-sdks-go-container"
-project(":beam-sdks-go-container").dir = file("sdks/go/container")
-include "beam-sdks-go-examples"
-project(":beam-sdks-go-examples").dir = file("sdks/go/examples")
-include "beam-sdks-go-test"
-project(":beam-sdks-go-test").dir = file("sdks/go/test")
-include "beam-sdks-java-build-tools"
-project(":beam-sdks-java-build-tools").dir = file("sdks/java/build-tools")
-include "beam-sdks-java-container"
-project(":beam-sdks-java-container").dir = file("sdks/java/container")
-include "beam-sdks-java-core"
-project(":beam-sdks-java-core").dir = file("sdks/java/core")
-include "beam-sdks-java-extensions-euphoria"
-project(":beam-sdks-java-extensions-euphoria").dir = file("sdks/java/extensions/euphoria")
-include "beam-sdks-java-extensions-kryo"
-project(":beam-sdks-java-extensions-kryo").dir = file("sdks/java/extensions/kryo")
-include "beam-sdks-java-extensions-google-cloud-platform-core"
-project(":beam-sdks-java-extensions-google-cloud-platform-core").dir = file("sdks/java/extensions/google-cloud-platform-core")
-include "beam-sdks-java-extensions-json-jackson"
-project(":beam-sdks-java-extensions-json-jackson").dir = file("sdks/java/extensions/jackson")
-include "beam-sdks-java-extensions-join-library"
-project(":beam-sdks-java-extensions-join-library").dir = file("sdks/java/extensions/join-library")
-include "beam-sdks-java-extensions-protobuf"
-project(":beam-sdks-java-extensions-protobuf").dir = file("sdks/java/extensions/protobuf")
-include "beam-sdks-java-extensions-sketching"
-project(":beam-sdks-java-extensions-sketching").dir = file("sdks/java/extensions/sketching")
-include "beam-sdks-java-extensions-sorter"
-project(":beam-sdks-java-extensions-sorter").dir = file("sdks/java/extensions/sorter")
-include "beam-sdks-java-extensions-sql"
-project(":beam-sdks-java-extensions-sql").dir = file("sdks/java/extensions/sql")
-include "beam-sdks-java-extensions-sql-jdbc"
-project(":beam-sdks-java-extensions-sql-jdbc").dir = file("sdks/java/extensions/sql/jdbc")
-include "beam-sdks-java-extensions-sql-shell"
-project(":beam-sdks-java-extensions-sql-shell").dir = file("sdks/java/extensions/sql/shell")
-include "beam-sdks-java-extensions-sql-hcatalog"
-project(":beam-sdks-java-extensions-sql-hcatalog").dir = file("sdks/java/extensions/sql/hcatalog")
-include "beam-sdks-java-extensions-sql-datacatalog"
-project(":beam-sdks-java-extensions-sql-datacatalog").dir = file("sdks/java/extensions/sql/datacatalog")
-include "beam-sdks-java-fn-execution"
-project(":beam-sdks-java-fn-execution").dir = file("sdks/java/fn-execution")
-include "beam-sdks-java-harness"
-project(":beam-sdks-java-harness").dir = file("sdks/java/harness")
-include "beam-sdks-java-io-amazon-web-services"
-project(":beam-sdks-java-io-amazon-web-services").dir = file("sdks/java/io/amazon-web-services")
-include "beam-sdks-java-io-amqp"
-project(":beam-sdks-java-io-amqp").dir = file("sdks/java/io/amqp")
-include "beam-sdks-java-io-cassandra"
-project(":beam-sdks-java-io-cassandra").dir = file("sdks/java/io/cassandra")
-include "beam-sdks-java-io-clickhouse"
-project(":beam-sdks-java-io-clickhouse").dir = file("sdks/java/io/clickhouse")
-include "beam-sdks-java-io-common"
-project(":beam-sdks-java-io-common").dir = file("sdks/java/io/common")
-include "beam-sdks-java-io-elasticsearch"
-project(":beam-sdks-java-io-elasticsearch").dir = file("sdks/java/io/elasticsearch")
-include "beam-sdks-java-io-elasticsearch-tests-2"
-project(":beam-sdks-java-io-elasticsearch-tests-2").dir = file("sdks/java/io/elasticsearch-tests/elasticsearch-tests-2")
-include "beam-sdks-java-io-elasticsearch-tests-5"
-project(":beam-sdks-java-io-elasticsearch-tests-5").dir = file("sdks/java/io/elasticsearch-tests/elasticsearch-tests-5")
-include "beam-sdks-java-io-elasticsearch-tests-6"
-project(":beam-sdks-java-io-elasticsearch-tests-6").dir = file("sdks/java/io/elasticsearch-tests/elasticsearch-tests-6")
-include "beam-sdks-java-io-elasticsearch-tests-common"
-project(":beam-sdks-java-io-elasticsearch-tests-common").dir = file("sdks/java/io/elasticsearch-tests/elasticsearch-tests-common")
-include "beam-sdks-java-io-file-based-io-tests"
-project(":beam-sdks-java-io-file-based-io-tests").dir = file("sdks/java/io/file-based-io-tests")
-include "beam-sdks-java-io-google-cloud-platform"
-project(":beam-sdks-java-io-google-cloud-platform").dir = file("sdks/java/io/google-cloud-platform")
-include "beam-sdks-java-io-hadoop-common"
-project(":beam-sdks-java-io-hadoop-common").dir = file("sdks/java/io/hadoop-common")
-include "beam-sdks-java-io-hadoop-file-system"
-project(":beam-sdks-java-io-hadoop-file-system").dir = file("sdks/java/io/hadoop-file-system")
-include "beam-sdks-java-io-hadoop-format"
-project(":beam-sdks-java-io-hadoop-format").dir = file("sdks/java/io/hadoop-format")
-include "beam-sdks-java-io-hbase"
-project(":beam-sdks-java-io-hbase").dir = file("sdks/java/io/hbase")
-include "beam-sdks-java-io-hcatalog"
-project(":beam-sdks-java-io-hcatalog").dir = file("sdks/java/io/hcatalog")
-include "beam-sdks-java-io-jdbc"
-project(":beam-sdks-java-io-jdbc").dir = file("sdks/java/io/jdbc")
-include "beam-sdks-java-io-jms"
-project(":beam-sdks-java-io-jms").dir = file("sdks/java/io/jms")
-include "beam-sdks-java-io-kafka"
-project(":beam-sdks-java-io-kafka").dir = file("sdks/java/io/kafka")
-include "beam-sdks-java-io-kinesis"
-project(":beam-sdks-java-io-kinesis").dir = file("sdks/java/io/kinesis")
-include "beam-sdks-java-io-kudu"
-project(":beam-sdks-java-io-kudu").dir = file("sdks/java/io/kudu")
-include "beam-sdks-java-io-mongodb"
-project(":beam-sdks-java-io-mongodb").dir = file("sdks/java/io/mongodb")
-include "beam-sdks-java-io-mqtt"
-project(":beam-sdks-java-io-mqtt").dir = file("sdks/java/io/mqtt")
-include "beam-sdks-java-io-parquet"
-project(":beam-sdks-java-io-parquet").dir = file("sdks/java/io/parquet")
-include "beam-sdks-java-io-rabbitmq"
-project(":beam-sdks-java-io-rabbitmq").dir = file("sdks/java/io/rabbitmq")
-include "beam-sdks-java-io-redis"
-project(":beam-sdks-java-io-redis").dir = file("sdks/java/io/redis")
-include "beam-sdks-java-io-solr"
-project(":beam-sdks-java-io-solr").dir = file("sdks/java/io/solr")
-include "beam-sdks-java-io-tika"
-project(":beam-sdks-java-io-tika").dir = file("sdks/java/io/tika")
-include "beam-sdks-java-io-xml"
-project(":beam-sdks-java-io-xml").dir = file("sdks/java/io/xml")
-include "beam-sdks-java-io-synthetic"
-project(":beam-sdks-java-io-synthetic").dir = file("sdks/java/io/synthetic")
-include "beam-sdks-java-javadoc"
-project(":beam-sdks-java-javadoc").dir = file("sdks/java/javadoc")
-include "beam-sdks-java-load-tests"
-project(":beam-sdks-java-load-tests").dir = file("sdks/java/testing/load-tests")
-include "beam-sdks-java-maven-archetypes-examples"
-project(":beam-sdks-java-maven-archetypes-examples").dir = file("sdks/java/maven-archetypes/examples")
-include "beam-sdks-java-maven-archetypes-starter"
-project(":beam-sdks-java-maven-archetypes-starter").dir = file("sdks/java/maven-archetypes/starter")
-include "beam-sdks-java-nexmark"
-project(":beam-sdks-java-nexmark").dir = file("sdks/java/testing/nexmark")
-include "beam-sdks-python"
-project(":beam-sdks-python").dir = file("sdks/python")
-include "beam-sdks-python-container"
-project(":beam-sdks-python-container").dir = file("sdks/python/container")
-include "beam-sdks-python-container-py3"
-project(":beam-sdks-python-container-py3").dir = file("sdks/python/container/py3")
-include "beam-sdks-python-test-suites-dataflow"
-project(":beam-sdks-python-test-suites-dataflow").dir = file("sdks/python/test-suites/dataflow")
-include "beam-sdks-python-test-suites-dataflow-py35"
-project(":beam-sdks-python-test-suites-dataflow-py35").dir = file("sdks/python/test-suites/dataflow/py35")
-include "beam-sdks-python-test-suites-dataflow-py36"
-project(":beam-sdks-python-test-suites-dataflow-py36").dir = file("sdks/python/test-suites/dataflow/py36")
-include "beam-sdks-python-test-suites-direct-py35"
-project(":beam-sdks-python-test-suites-direct-py35").dir = file("sdks/python/test-suites/direct/py35")
-include "beam-sdks-python-test-suites-direct-py36"
-project(":beam-sdks-python-test-suites-direct-py36").dir = file("sdks/python/test-suites/direct/py36")
-include "beam-sdks-python-test-suites-tox-py35"
-project(":beam-sdks-python-test-suites-tox-py35").dir = file("sdks/python/test-suites/tox/py35")
-include "beam-sdks-python-test-suites-tox-py36"
-project(":beam-sdks-python-test-suites-tox-py36").dir = file("sdks/python/test-suites/tox/py36")
-include "beam-sdks-python-test-suites-tox-py37"
-project(":beam-sdks-python-test-suites-tox-py37").dir = file("sdks/python/test-suites/tox/py37")
-include "beam-sdks-python-load-tests"
-project(":beam-sdks-python-load-tests").dir = file("sdks/python/apache_beam/testing/load_tests")
-include "beam-vendor-grpc-1_13_1"
-project(":beam-vendor-grpc-1_13_1").dir = file("vendor/grpc-1_13_1")
-include "beam-sdks-java-test-utils"
-project(":beam-sdks-java-test-utils").dir = file("sdks/java/testing/test-utils")
-include "beam-vendor-sdks-java-extensions-protobuf"
-project(":beam-vendor-sdks-java-extensions-protobuf").dir = file("vendor/sdks-java-extensions-protobuf")
-include "beam-vendor-guava-20_0"
-project(":beam-vendor-guava-20_0").dir = file("vendor/guava-20_0")
-include "beam-website"
-project(":beam-website").dir = file("website")
-include "beam-runners-google-cloud-dataflow-java-legacy-worker"
-project(":beam-runners-google-cloud-dataflow-java-legacy-worker").dir = file("runners/google-cloud-dataflow-java/worker/legacy-worker")
-include "beam-runners-google-cloud-dataflow-java-fn-api-worker"
-project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").dir = file("runners/google-cloud-dataflow-java/worker")
-include "beam-runners-google-cloud-dataflow-java-windmill"
-project(":beam-runners-google-cloud-dataflow-java-windmill").dir = file("runners/google-cloud-dataflow-java/worker/windmill")
+include ":runners:gearpump"
+include ":runners:google-cloud-dataflow-java"
+include ":runners:google-cloud-dataflow-java:examples"
+include ":runners:google-cloud-dataflow-java:examples-streaming"
+include ":runners:java-fn-execution"
+include ":runners:local-java"
+include ":runners:reference:java"
+include ":runners:reference:job-server"
+include ":runners:spark"
+include ":runners:spark:job-server"
+include ":runners:samza"
+include ":runners:samza:job-server"
+include ":sdks:go"
+include ":sdks:go:container"
+include ":sdks:go:examples"
+include ":sdks:go:test"
+include ":sdks:java:bom"
+include ":sdks:java:build-tools"
+include ":sdks:java:container"
+include ":sdks:java:core"
+include ":sdks:java:extensions:euphoria"
+include ":sdks:java:extensions:kryo"
+include ":sdks:java:extensions:google-cloud-platform-core"
+include ":sdks:java:extensions:jackson"
+include ":sdks:java:extensions:join-library"
+include ":sdks:java:extensions:protobuf"
+include ":sdks:java:extensions:sketching"
+include ":sdks:java:extensions:sorter"
+include ":sdks:java:extensions:sql"
+include ":sdks:java:extensions:sql:jdbc"
+include ":sdks:java:extensions:sql:shell"
+include ":sdks:java:extensions:sql:hcatalog"
+include ":sdks:java:extensions:sql:datacatalog"
+include ":sdks:java:fn-execution"
+include ":sdks:java:harness"
+include ":sdks:java:io:amazon-web-services"
+include ":sdks:java:io:amqp"
+include ":sdks:java:io:cassandra"
+include ":sdks:java:io:clickhouse"
+include ":sdks:java:io:common"
+include ":sdks:java:io:elasticsearch"
+include ":sdks:java:io:elasticsearch-tests:elasticsearch-tests-2"
+include ":sdks:java:io:elasticsearch-tests:elasticsearch-tests-5"
+include ":sdks:java:io:elasticsearch-tests:elasticsearch-tests-6"
+include ":sdks:java:io:elasticsearch-tests:elasticsearch-tests-common"
+include ":sdks:java:io:file-based-io-tests"
+include ":sdks:java:io:google-cloud-platform"
+include ":sdks:java:io:hadoop-common"
+include ":sdks:java:io:hadoop-file-system"
+include ":sdks:java:io:hadoop-format"
+include ":sdks:java:io:hbase"
+include ":sdks:java:io:hcatalog"
+include ":sdks:java:io:jdbc"
+include ":sdks:java:io:jms"
+include ":sdks:java:io:kafka"
+include ":sdks:java:io:kinesis"
+include ":sdks:java:io:kudu"
+include ":sdks:java:io:mongodb"
+include ":sdks:java:io:mqtt"
+include ":sdks:java:io:parquet"
+include ":sdks:java:io:rabbitmq"
+include ":sdks:java:io:redis"
+include ":sdks:java:io:solr"
+include ":sdks:java:io:tika"
+include ":sdks:java:io:xml"
+include ":sdks:java:io:synthetic"
+include ":sdks:java:javadoc"
+include ":sdks:java:testing:load-tests"
+include ":sdks:java:maven-archetypes:examples"
+include ":sdks:java:maven-archetypes:starter"
+include ":sdks:java:testing:nexmark"
+include ":sdks:python"
+include ":sdks:python:apache_beam:testing:load_tests"
+include ":sdks:python:container"
+include ":sdks:python:container:py3"
+include ":sdks:python:test-suites:dataflow"
+include ":sdks:python:test-suites:dataflow:py35"
+include ":sdks:python:test-suites:dataflow:py36"
+include ":sdks:python:test-suites:direct:py35"
+include ":sdks:python:test-suites:direct:py36"
+include ":sdks:python:test-suites:tox:py35"
+include ":sdks:python:test-suites:tox:py36"
+include ":sdks:python:test-suites:tox:py37"
+include ":vendor:grpc-1_13_1"
+include ":sdks:java:testing:test-utils"
+include ":vendor:sdks-java-extensions-protobuf"
+include ":vendor:guava-20_0"
+include ":website"
+include ":runners:google-cloud-dataflow-java:worker:legacy-worker"
+include ":runners:google-cloud-dataflow-java:worker"
+include ":runners:google-cloud-dataflow-java:worker:windmill"
+// no dots allowed for project paths
 include "beam-test-infra-metrics"
 project(":beam-test-infra-metrics").dir = file(".test-infra/metrics")
-include "beam-sdks-java-bom"
-project(":beam-sdks-java-bom").dir = file("sdks/java/bom")
 include "beam-test-tools"
-project(":beam-test-tools").dir = file(".test-infra/tools")
+project(":beam-test-tools").dir = file(".test-infra/tools")
\ No newline at end of file
diff --git a/vendor/sdks-java-extensions-protobuf/build.gradle b/vendor/sdks-java-extensions-protobuf/build.gradle
index fc0d7507ede7c..d0835379707c4 100644
--- a/vendor/sdks-java-extensions-protobuf/build.gradle
+++ b/vendor/sdks-java-extensions-protobuf/build.gradle
@@ -46,14 +46,14 @@ ext.summary = "Add support to Apache Beam for Vendored Google Protobuf."
  * the following projects are evaluated before we evaluate this project. This is because
  * we are attempting to reference the "sourceSets.main.java.srcDirs" directly.
  */
-evaluationDependsOn(":beam-sdks-java-extensions-protobuf")
+evaluationDependsOn(":sdks:java:extensions:protobuf")
 
 compileJava {
-    source project(":beam-sdks-java-extensions-protobuf").sourceSets.main.java.srcDirs
+    source project(":sdks:java:extensions:protobuf").sourceSets.main.java.srcDirs
 }
 
 dependencies {
     compile 'com.google.guava:guava:20.0'
     compile 'com.google.protobuf:protobuf-java:3.5.1'
-    shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
+    shadow project(path: ":sdks:java:core", configuration: "shadow")
 }
diff --git a/website/README.md b/website/README.md
index 544c911eff27f..659592bb0f1b4 100644
--- a/website/README.md
+++ b/website/README.md
@@ -37,13 +37,13 @@ run website tests.
 
 The following command is used to build and serve the website locally.
 
-    $ ./gradlew :beam-website:serveWebsite
+    $ ./gradlew :website:serveWebsite
 
 Any changes made locally will trigger a rebuild of the website.
 
 Websites tests may be run using this command:
 
-    $ ./gradlew :beam-website:testWebsite
+    $ ./gradlew :website:testWebsite
 
 ## Website push
 
diff --git a/website/_config_test.yml b/website/_config_test.yml
index 08bd1942cb4d4..06e06696bf126 100644
--- a/website/_config_test.yml
+++ b/website/_config_test.yml
@@ -12,7 +12,7 @@
 # Welcome to Jekyll!
 
 # This config file specifies a set of config overrides necessary to stage
-# the website on GCS as part of the :beam-website:stageWebsite task. This is
+# the website on GCS as part of the :website:stageWebsite task. This is
 # run automatically on Jenkins for website pull requests to make it easy to
 # preview website changes during code review.
 
diff --git a/website/build.gradle b/website/build.gradle
index 6045ef585cdfa..9beac9c565453 100644
--- a/website/build.gradle
+++ b/website/build.gradle
@@ -341,7 +341,7 @@ publishWebsite.dependsOn commitWebsite
 /*
  * Stages a pull request on GCS
  * For example:
- *   ./gradlew :beam-website:stageWebsite -PwebsiteBucket=foo
+ *   ./gradlew :website:stageWebsite -PwebsiteBucket=foo
  */
 task stageWebsite {
   doLast {
diff --git a/website/src/contribute/release-guide.md b/website/src/contribute/release-guide.md
index 0a7401c2688d9..49c22462e4833 100644
--- a/website/src/contribute/release-guide.md
+++ b/website/src/contribute/release-guide.md
@@ -779,31 +779,31 @@ _Note_: -Prepourl and -Pver can be found in the RC vote email sent by Release Ma
 
   Direct Runner:
   ```
-  ./gradlew :beam-runners-direct-java:runQuickstartJavaDirect \ 
+  ./gradlew :runners:direct-java:runQuickstartJavaDirect \
   -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \
   -Pver=${RELEASE_VERSION}
   ```
   Apex Local Runner
   ```
-  ./gradlew :beam-runners-apex:runQuickstartJavaApex \
+  ./gradlew :runners:apex:runQuickstartJavaApex \
   -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam${KEY} \
   -Pver=${RELEASE_VERSION}
   ```
   Flink Local Runner
   ```
-  ./gradlew :beam-runners-flink_2.11:runQuickstartJavaFlinkLocal \
+  ./gradlew :runners:flink:1.5:runQuickstartJavaFlinkLocal \
   -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \
   -Pver=${RELEASE_VERSION}
   ```
   Spark Local Runner
   ```
-  ./gradlew :beam-runners-spark:runQuickstartJavaSpark \
+  ./gradlew :runners:spark:runQuickstartJavaSpark \
   -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \
   -Pver=${RELEASE_VERSION}
   ```
   Dataflow Runner
   ```
-  ./gradlew :beam-runners-google-cloud-dataflow-java:runQuickstartJavaDataflow \
+  ./gradlew :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow \
   -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \
   -Pver= ${RELEASE_VERSION}\
   -PgcpProject=${YOUR_GCP_PROJECT} \
@@ -831,7 +831,7 @@ _Note_: -Prepourl and -Pver can be found in the RC vote email sent by Release Ma
     ```
   Run 
   ```
-  ./gradlew :beam-runners-google-cloud-dataflow-java:runMobileGamingJavaDataflow \
+  ./gradlew :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow \
    -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \ 
    -Pver= ${RELEASE_VERSION}\
    -PgcpProject=${YOUR_GCP_PROJECT} \
diff --git a/website/src/documentation/dsls/sql/shell.md b/website/src/documentation/dsls/sql/shell.md
index 4ef670a415896..dae4939eb0116 100644
--- a/website/src/documentation/dsls/sql/shell.md
+++ b/website/src/documentation/dsls/sql/shell.md
@@ -31,7 +31,7 @@ This page describes how to work with the shell, but does not focus on specific f
 To use Beam SQL shell, you must first clone the [Beam SDK repository](https://github.com/apache/beam). Then, from the root of the repository clone, execute the following commands to run the shell:
 
 ```
-./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':beam-runners-flink_2.11,:beam-sdks-java-io-kafka' installDist
+./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':runners:flink:1.5,:sdks:java:io:kafka' installDist
 
 ./sdks/java/extensions/sql/shell/build/install/beam-sdks-java-extensions-sql-shell/bin/beam-sdks-java-extensions-sql-shell
 ```
@@ -119,7 +119,7 @@ By default, Beam uses the `DirectRunner` to run the pipeline on the machine wher
 1.  Make sure the SQL shell includes the desired runner. Add the corresponding project id to the `-Pbeam.sql.shell.bundled` parameter of the Gradle invocation ([source code](https://github.com/apache/beam/blob/master/sdks/java/extensions/sql/shell/build.gradle), [project ids](https://github.com/apache/beam/blob/master/settings.gradle)). For example, use the following command to include Flink runner and KafkaIO:
 
     ```
-    ./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':beam-runners-flink_2.11,:beam-sdks-java-io-kafka' installDist
+    ./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':runners:flink:1.5,:sdks:java:io:kafka' installDist
     ```
 
     _Note: You can bundle multiple runners (using a comma-separated list) or other additional components in the same manner. For example, you can add support for more I/Os._
@@ -145,7 +145,7 @@ To configure the runner, you must specify `PipelineOptions` by using the `SET` c
 You can also build your own standalone package for SQL shell using `distZip` or `distTar` tasks. For example:
 
 ```
-./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':beam-runners-flink_2.11,:beam-sdks-java-io-kafka' distZip
+./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':runners:flink:1.5,:sdks:java:io:kafka' distZip
 
 ls ./sdks/java/extensions/sql/shell/build/distributions/
 beam-sdks-java-extensions-sql-shell-2.6.0-SNAPSHOT.tar beam-sdks-java-extensions-sql-shell-2.6.0-SNAPSHOT.zip
diff --git a/website/src/documentation/runners/flink.md b/website/src/documentation/runners/flink.md
index d33c8b50fb191..53d51cf813e77 100644
--- a/website/src/documentation/runners/flink.md
+++ b/website/src/documentation/runners/flink.md
@@ -245,10 +245,10 @@ download it on the [Downloads page]({{ site.baseurl
 available.
 
 
-1. *Only required once:* Build the SDK harness container: `./gradlew :beam-sdks-python-container:docker`
+1. *Only required once:* Build the SDK harness container: `./gradlew :sdks:python:container:docker`
 
 
-2. Start the JobService endpoint: `./gradlew :beam-runners-flink_2.11-job-server:runShadow`
+2. Start the JobService endpoint: `./gradlew :runners:flink:1.5:job-server:runShadow`
 
 
 
@@ -278,7 +278,7 @@ To run on a separate [Flink cluster](https://ci.apache.org/projects/flink/flink-
 1. Start a Flink cluster which exposes the Rest interface on `localhost:8081` by default.
 
 
-2. Start JobService with Flink Rest endpoint: `./gradlew :beam-runners-flink_2.11-job-server:runShadow -PflinkMasterUrl=localhost:8081`.
+2. Start JobService with Flink Rest endpoint: `./gradlew :runners:flink:1.5:job-server:runShadow -PflinkMasterUrl=localhost:8081`.
 
 
 3. Submit the pipeline as above.
diff --git a/website/src/documentation/sdks/euphoria.md b/website/src/documentation/sdks/euphoria.md
index 652708b116a1d..3819c13f8224a 100644
--- a/website/src/documentation/sdks/euphoria.md
+++ b/website/src/documentation/sdks/euphoria.md
@@ -126,12 +126,12 @@ The operator consumes `input`, it applies given lambda expression (`String::valu
 First step to build any operator is to give it a name through `named()` method. The name is propagated through system and can latter be used when debugging.
 
 ### Coders and Types
-Beam's Java SDK requires developers to supply `Coder` for custom element type in order to have a way of materializing elements. Euphoria allows to use [Kryo](https://github.com/EsotericSoftware/kryo) as a way of serialization. The [Kryo](https://github.com/EsotericSoftware/kryo) is located in `:beam-sdks-java-extensions-kryo` module.
+Beam's Java SDK requires developers to supply `Coder` for custom element type in order to have a way of materializing elements. Euphoria allows to use [Kryo](https://github.com/EsotericSoftware/kryo) as a way of serialization. The [Kryo](https://github.com/EsotericSoftware/kryo) is located in `:sdks:java:extensions:kryo` module.
 
 ```groovy
 //gradle
 dependencies {
-    compile "org.apache.beam:beam-sdks-java-extensions-kryo:${beam.version}"
+    compile "org.apache.beam:sdks:java:extensions:kryo:${beam.version}"
 }
 ```
 ```xml
diff --git a/website/src/documentation/sdks/nexmark.md b/website/src/documentation/sdks/nexmark.md
index bb3b9c6e7b997..d319a001ed0a7 100644
--- a/website/src/documentation/sdks/nexmark.md
+++ b/website/src/documentation/sdks/nexmark.md
@@ -148,8 +148,8 @@ When running via Gradle, the following two parameters control the execution:
         The command line to pass to the Nexmark main program.
 
     -P nexmark.runner
-	The Gradle project name of the runner, such as ":beam-runners-direct-java" or
-	":beam-runners-flink. The project names can be found in the root
+	The Gradle project name of the runner, such as ":runners:direct-java" or
+	":runners:flink:1.5. The project names can be found in the root
         `settings.gradle`.
 
 Test data is deterministically synthesized on demand. The test
@@ -498,8 +498,8 @@ SMOKE suite can make sure there is nothing broken in the Nexmark suite.
 
 Batch Mode:
 
-    ./gradlew :beam-sdks-java-nexmark:run \
-        -Pnexmark.runner=":beam-runners-direct-java" \
+    ./gradlew :sdks:java:testing:nexmark:run \
+        -Pnexmark.runner=":runners:direct-java" \
         -Pnexmark.args="
             --runner=DirectRunner
             --streaming=false
@@ -511,8 +511,8 @@ Batch Mode:
 
 Streaming Mode:
 
-    ./gradlew :beam-sdks-java-nexmark:run \
-        -Pnexmark.runner=":beam-runners-direct-java" \
+    ./gradlew :sdks:java:testing:nexmark:run \
+        -Pnexmark.runner=":runners:direct-java" \
         -Pnexmark.args="
             --runner=DirectRunner
             --streaming=true
@@ -530,8 +530,8 @@ configure logging.
 
 Batch Mode:
 
-    ./gradlew :beam-sdks-java-nexmark:run \
-        -Pnexmark.runner=":beam-runners-spark" \
+    ./gradlew :sdks:java:testing:nexmark:run \
+        -Pnexmark.runner=":runners:spark" \
         -Pnexmark.args="
             --runner=SparkRunner
             --suite=SMOKE
@@ -542,8 +542,8 @@ Batch Mode:
 
 Streaming Mode:
 
-    ./gradlew :beam-sdks-java-nexmark:run \
-        -Pnexmark.runner=":beam-runners-spark" \
+    ./gradlew :sdks:java:testing:nexmark:run \
+        -Pnexmark.runner=":runners:spark" \
         -Pnexmark.args="
             --runner=SparkRunner
             --suite=SMOKE
@@ -556,8 +556,8 @@ Streaming Mode:
 
 Batch Mode:
 
-    ./gradlew :beam-sdks-java-nexmark:run \
-        -Pnexmark.runner=":beam-runners-flink_2.11" \
+    ./gradlew :sdks:java:testing:nexmark:run \
+        -Pnexmark.runner=":runners:flink:1.5" \
         -Pnexmark.args="
             --runner=FlinkRunner
             --suite=SMOKE
@@ -569,8 +569,8 @@ Batch Mode:
 
 Streaming Mode:
 
-    ./gradlew :beam-sdks-java-nexmark:run \
-        -Pnexmark.runner=":beam-runners-flink_2.11" \
+    ./gradlew :sdks:java:testing:nexmark:run \
+        -Pnexmark.runner=":runners:flink:1.5" \
         -Pnexmark.args="
             --runner=FlinkRunner
             --suite=SMOKE
@@ -584,8 +584,8 @@ Streaming Mode:
 
 Batch Mode:
 
-    ./gradlew :beam-sdks-java-nexmark:run \
-        -Pnexmark.runner=":beam-runners-apex" \
+    ./gradlew :sdks:java:testing:nexmark:run \
+        -Pnexmark.runner=":runners:apex" \
         -Pnexmark.args="
             --runner=ApexRunner
             --suite=SMOKE
@@ -596,8 +596,8 @@ Batch Mode:
 
 Streaming Mode:
 
-    ./gradlew :beam-sdks-java-nexmark:run \
-        -Pnexmark.runner=":beam-runners-apex" \
+    ./gradlew :sdks:java:testing:nexmark:run \
+        -Pnexmark.runner=":runners:apex" \
         -Pnexmark.args="
             --runner=ApexRunner
             --suite=SMOKE
@@ -617,7 +617,7 @@ Set these up first so the below command is valid
 
 Launch:
 
-    ./gradlew :beam-sdks-java-nexmark:run \
+    ./gradlew :sdks:java:testing:nexmark:run \
         -Pnexmark.runner=":beam-runners-google-cloud-dataflow" \
         -Pnexmark.args="
             --runner=DataflowRunner
@@ -659,7 +659,7 @@ Launch:
 
 Building package:
 
-    ./gradlew :beam-sdks-java-nexmark:assemble
+    ./gradlew :sdks:java:testing:nexmark:assemble
 
 Submit to the cluster:
 
diff --git a/website/src/roadmap/portability.md b/website/src/roadmap/portability.md
index fe1844ad5ce0a..bb22c0f4cd6af 100644
--- a/website/src/roadmap/portability.md
+++ b/website/src/roadmap/portability.md
@@ -155,11 +155,11 @@ for details.
 Currently, the Flink and Spark runners support portable pipeline execution.
 To run a basic Python wordcount (in batch mode) with embedded Flink or Spark:
 
-1. Run once to build the SDK harness container: `./gradlew :beam-sdks-python-container:docker`
+1. Run once to build the SDK harness container: `./gradlew :sdks:python:container:docker`
 2. Choose one:
- * Start the Flink portable JobService endpoint: `./gradlew :beam-runners-flink_2.11-job-server:runShadow`
- * Or start the Spark portable JobService endpoint: `./gradlew :beam-runners-spark-job-server:runShadow`
-3. Submit the wordcount pipeline to above endpoint: `./gradlew :beam-sdks-python:portableWordCount -PjobEndpoint=localhost:8099 -PenvironmentType=LOOPBACK`
+ * Start the Flink portable JobService endpoint: `./gradlew :runners:flink:1.5:job-server:runShadow`
+ * Or start the Spark portable JobService endpoint: `./gradlew :runners:spark:job-server:runShadow`
+3. Submit the wordcount pipeline to above endpoint: `./gradlew :sdks:python:portableWordCount -PjobEndpoint=localhost:8099 -PenvironmentType=LOOPBACK`
 
 To run the pipeline in streaming mode (currently only supported on Flink): `./gradlew :beam-sdks-python:portableWordCount -PjobEndpoint=localhost:8099 -Pstreaming`