apache · lukecwik · Mar 28, 2018 · Mar 27, 2018 · Mar 27, 2018 · Mar 28, 2018
diff --git a/build_rules.gradle b/build_rules.gradle
@@ -489,6 +489,41 @@ ext.applyDockerNature = {
   }
 }
 
+// A class defining the set of configurable properties accepted by containerImageName.
+class ContainerImageNameConfiguration {
+  String root = null // Sets the docker repository root (optional).
+  String name = null // Sets the short container image name, such as "go" (required).
+  String tag  = null // Sets the image tag (optional).
+}
+
+// containerImageName returns a configurable container image name, by default a
+// development image at bintray.io (see sdks/CONTAINERS.md):
+//
+//     $USER-docker-apache.bintray.io/beam/$NAME:latest
+//
+// Both the root and tag can be defined using properties or explicitly provided.
+ext.containerImageName = {
+  println "containerImageName with " + (it ? "$it" : "default configuration") + " for project $project.name"
+  // Use the implicit it parameter of the closure to handle zero argument or one argument map calls.
+  ContainerImageNameConfiguration configuration = it ? it as ContainerImageNameConfiguration : new ContainerImageNameConfiguration()
+
+  if (configuration.root == null) {
+    if (rootProject.hasProperty(["docker-repository-root"])) {
+      configuration.root = rootProject["docker-repository-root"]
+    } else {
+      configuration.root = "${System.properties["user.name"]}-docker-apache.bintray.io/beam"
+    }
+  }
+  if (configuration.tag == null) {
+    if (rootProject.hasProperty(["docker-tag"])) {
+      configuration.tag = rootProject["docker-tag"]
+    } else {
+      configuration.tag = 'latest'
+    }
+  }
+  return "${configuration.root}/${configuration.name}:${configuration.tag}"
+}
+
 /*************************************************************************************************/
 
 ext.applyGrpcNature = {

diff --git a/runners/gcp/gcemd/build.gradle b/runners/gcp/gcemd/build.gradle
@@ -18,6 +18,7 @@
 
 apply from: project(":").file("build_rules.gradle")
 applyGoNature()
+applyDockerNature()
 
 description = "Apache Beam :: Runners :: Google Cloud Platform :: GCE metadata provisioning"
 
@@ -42,3 +43,10 @@ golang {
     outputLocation = './build/target/${GOOS}_${GOARCH}/gcemd'
   }
 }
+
+docker {
+  name containerImageName(name: "gcemd")
+  files "./build/"
+}
+// Ensure that making the docker image builds any required artifacts
+dockerPrepare.dependsOn build
diff --git a/runners/gcp/gcsproxy/build.gradle b/runners/gcp/gcsproxy/build.gradle
@@ -18,6 +18,7 @@
 
 apply from: project(":").file("build_rules.gradle")
 applyGoNature()
+applyDockerNature()
 
 description = "Apache Beam :: Runners :: Google Cloud Platform :: GCS artifact proxy"
 
@@ -42,3 +43,10 @@ golang {
     outputLocation = './build/target/${GOOS}_${GOARCH}/gcsproxy'
   }
 }
+
+docker {
+  name containerImageName(name: "gcsproxy")
+  files "./build/"
+}
+// Ensure that making the docker image builds any required artifacts
+dockerPrepare.dependsOn build
diff --git a/sdks/CONTAINERS.md b/sdks/CONTAINERS.md
@@ -38,57 +38,65 @@ current username with a docker repository named "apache".
 platform. You can verify that it works by running `docker images` or any other
 docker command.
 
-Run Maven with the `build-containers` profile:
+Run Gradle with the `docker` target:
 
 ```
 $ pwd
 [...]/beam
-$ mvn clean install -DskipTests -Pbuild-containers
+$ ./gradlew docker
 [...]
-[INFO] --- dockerfile-maven-plugin:1.3.5:build (default) @ beam-sdks-python-container ---
-[INFO] Using Google application default credentials
-[INFO] loaded credentials for user account with clientId=[...].apps.googleusercontent.com
-[INFO] Building Docker context /Users/herohde/go/src/github.com/apache/beam/sdks/python/container
-[INFO] 
-[INFO] Image will be built as herohde-docker-apache.bintray.io/beam/python:latest
-[INFO] 
-[INFO] Step 1/4 : FROM python:2
-[INFO] Pulling from library/python
-[INFO] Digest: sha256:181ee8edfd9d44323c82dcba0b187f1ee2eb3d4a11c8398fc06952ed5f9ef32c
-[INFO] Status: Image is up to date for python:2
-[INFO]  ---> b1d5c2d7dda8
-[INFO] Step 2/4 : MAINTAINER "Apache Beam <dev@beam.apache.org>"
-[INFO]  ---> Running in f1bc3c4943b3
-[INFO]  ---> 9867b512e47e
-[INFO] Removing intermediate container f1bc3c4943b3
-[INFO] Step 3/4 : ADD target/linux_amd64/boot /opt/apache/beam/
-[INFO]  ---> 5cb81c3d2d90
-[INFO] Removing intermediate container 4a41ad80005a
-[INFO] Step 4/4 : ENTRYPOINT /opt/apache/beam/boot
-[INFO]  ---> Running in 40f5b945afe7
-[INFO]  ---> c8bf712741c8
-[INFO] Removing intermediate container 40f5b945afe7
-[INFO] Successfully built c8bf712741c8
-[INFO] Successfully tagged herohde-docker-apache.bintray.io/beam/python:latest
-[INFO] 
-[INFO] Detected build of image with id c8bf712741c8
-[INFO] Building jar: /Users/herohde/go/src/github.com/apache/beam/sdks/python/container/target/beam-sdks-python-container-2.3.0-SNAPSHOT-docker-info.jar
-[INFO] Successfully built herohde-docker-apache.bintray.io/beam/python:latest
-[INFO]
+> Task :sdks:python:container:docker 
+a571bb44bc32: Verifying Checksum
+a571bb44bc32: Download complete
+aa6d783919f6: Verifying Checksum
+aa6d783919f6: Download complete
+f2b6b4884fc8: Verifying Checksum
+f2b6b4884fc8: Download complete
+f2b6b4884fc8: Pull complete
+74eaa8be7221: Pull complete
+2d6e98fe4040: Pull complete
+414666f7554d: Pull complete
+bb0bcc8d7f6a: Pull complete
+a571bb44bc32: Pull complete
+aa6d783919f6: Pull complete
+Digest: sha256:d9455be2cc68ded908084ec5b63a5cbb87f12ec0915c2f146751bd50b9aef01a
+Status: Downloaded newer image for python:2
+ ---> 2863c80c418c
+Step 2/6 : MAINTAINER "Apache Beam <dev@beam.apache.org>"
+ ---> Running in c787617f4af1
+Removing intermediate container c787617f4af1
+ ---> b4ffbbf94717
+[...]
+ ---> a77003ead1a1
+Step 5/6 : ADD target/linux_amd64/boot /opt/apache/beam/
+ ---> 4998013b3d63
+Step 6/6 : ENTRYPOINT ["/opt/apache/beam/boot"]
+ ---> Running in 30079dc4204b
+Removing intermediate container 30079dc4204b
+ ---> 4ea515403a1a
+Successfully built 4ea515403a1a
+Successfully tagged herohde-docker-apache.bintray.io/beam/python:latest
 [...]
 ```
 
 Note that the container images include built content, including the Go boot
-code, so you should build from the top level directory unless you're familiar
-with Maven.
+code. Some images, notably python, take a while to build, so building just
+the specific images needed can be a lot faster:
+
+```
+$ ./gradlew :sdks:java:container:docker
+$ ./gradlew :sdks:python:container:docker
+$ ./gradlew :sdks:go:containers:docker
+```
 
 **(Optional)** When built, you can see, inspect and run them locally:
 
 ```
 $ docker images
-REPOSITORY                                       TAG                    IMAGE ID            CREATED             SIZE
-herohde-docker-apache.bintray.io/beam/python     latest                 c8bf712741c8        About an hour ago   690MB
-herohde-docker-apache.bintray.io/beam/java       latest                 33efc0947952        About an hour ago   773MB
+REPOSITORY                                       TAG                    IMAGE ID            CREATED       SIZE
+herohde-docker-apache.bintray.io/beam/python     latest             4ea515403a1a      3 minutes ago     1.27GB
+herohde-docker-apache.bintray.io/beam/java       latest             0103512f1d8f     34 minutes ago      780MB
+herohde-docker-apache.bintray.io/beam/go         latest             ce055985808a     35 minutes ago      121MB
 [...]
 ```
 
@@ -100,14 +108,14 @@ images IDs will change.
 location. You can override it by adding: 
 
 ```
--Ddocker-repository-root=<location>
+-Pdocker-repository-root=<location>
 ```
 
 Similarly, if you want to specify a specific tag instead of "latest", such as a "2.3.0"
 version, you can do so by adding:
 
 ```
--Ddockerfile.tag=<tag>
+-Pdocker-tag=<tag>
 ```
 
 ## How to push container images
@@ -120,42 +128,45 @@ For the Python SDK harness container image, run:
 
 ```
 $ docker push $USER-docker-apache.bintray.io/beam/python:latest
-The push refers to a repository [herohde-docker-apache.bintray.io/beam/python]
-f2a8798331f5: Pushed 
-6b200cb2b684: Layer already exists 
-bf56c6510f38: Layer already exists 
-7890d67efa6f: Layer already exists 
-b456afdc9996: Layer already exists 
-d752a0310ee4: Layer already exists 
-db64edce4b5b: Layer already exists 
-d5d60fc34309: Layer already exists 
-c01c63c6823d: Layer already exists 
-latest: digest: sha256:58da4d9173a29622f0572cfa22dfeafc45e6750dde4beab57a47a9d1d17d601b size: 2222
-
+The push refers to repository [herohde-docker-apache.bintray.io/beam/python]
+723b66d57e21: Pushed 
+12d5806e6806: Pushed 
+b394bd077c6e: Pushed 
+ca82a2274c57: Pushed 
+de2fbb43bd2a: Pushed 
+4e32c2de91a6: Pushed 
+6e1b48dc2ccc: Pushed 
+ff57bdb79ac8: Pushed 
+6e5e20cbf4a7: Pushed 
+86985c679800: Pushed 
+8fad67424c4e: Pushed 
+latest: digest: sha256:86ad57055324457c3ea950f914721c596c7fa261c216efb881d0ca0bb8457535 size: 2646
 ```
 
-Similarly for the Java SDK harness container image. If you want to push the same image
-to multiple registries, you can retagging the image using `docker tag` and push.
+Similarly for the Java and Go SDK harness container images. If you want to push the same image
+to multiple registries, you can retag the image using `docker tag` and push.
 
 **(Optional)** On any machine, you can now pull the pushed container image:
 
 ```
 $ docker pull $USER-docker-apache.bintray.io/beam/python:latest
 latest: Pulling from beam/python
-85b1f47fba49: Pull complete 
-5409e9a7fa9e: Pull complete 
-661393707836: Pull complete 
-1bb98c08d57e: Pull complete 
-c842a08369e2: Pull complete 
-310408aa843f: Pull complete 
-d6a27cfc2cf1: Pull complete 
-7a24cf0c9043: Pull complete 
-290b127dfe35: Pull complete 
-Digest: sha256:58da4d9173a29622f0572cfa22dfeafc45e6750dde4beab57a47a9d1d17d601b
+f2b6b4884fc8: Pull complete 
+4fb899b4df21: Pull complete 
+74eaa8be7221: Pull complete 
+2d6e98fe4040: Pull complete 
+414666f7554d: Pull complete 
+bb0bcc8d7f6a: Pull complete 
+a571bb44bc32: Pull complete 
+aa6d783919f6: Pull complete 
+7255d71dee8f: Pull complete 
+08274803455d: Pull complete 
+ef79fab5686a: Pull complete 
+Digest: sha256:86ad57055324457c3ea950f914721c596c7fa261c216efb881d0ca0bb8457535
 Status: Downloaded newer image for herohde-docker-apache.bintray.io/beam/python:latest
 $ docker images
-REPOSITORY                                     TAG                 IMAGE ID            CREATED             SIZE
-herohde-docker-apache.bintray.io/beam/python   latest              c8bf712741c8        2 hours ago         690MB
+REPOSITORY                                     TAG                 IMAGE ID            CREATED          SIZE
+herohde-docker-apache.bintray.io/beam/python   latest          4ea515403a1a     35 minutes ago       1.27 GB
 [...]
 ```
 

diff --git a/sdks/go/build.gradle b/sdks/go/build.gradle
@@ -21,46 +21,15 @@ applyGoNature()
 
 description = "Apache Beam :: SDKs :: Go"
 
-def getLocalPlatform = {
-  String hostOs = com.github.blindpirate.gogradle.crossplatform.Os.getHostOs()
-  String hostArch = com.github.blindpirate.gogradle.crossplatform.Arch.getHostArch()
-  return hostOs + '-' + hostArch
-}
-
 golang {
   packagePath = 'github.com/apache/beam/sdks/go'
 
   build {
-    // We always want to build linux-amd64 in addition to the user host platform
-    // so we can submit this as the remote binary used within the Go container.
-    //
-    // TODO: Move this into build_rules.gradle applyGoNature. This currently
-    // can't be done because com.github.blindpirate.gogradle.crossplatform.*
-    // is not found within when invoked from within build_rules.gradle applyGoNature
-    targetPlatform = [getLocalPlatform(), 'linux-amd64']
-
     // The symlinks makes it hard (impossible?) to do a wildcard build
     // of pkg. Go build refuses to follow symlinks. Drop for now. The files
     // are built when tested anyway.
 
     // Build debugging utilities
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/beamctl github.com/apache/beam/sdks/go/cmd/beamctl'
-
-    // Build all the examples
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/autocomplete github.com/apache/beam/sdks/go/examples/complete/autocomplete'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/combine github.com/apache/beam/sdks/go/examples/cookbook/combine'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/contains github.com/apache/beam/sdks/go/examples/contains'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/debugging_wordcount github.com/apache/beam/sdks/go/examples/debugging_wordcount'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/filter github.com/apache/beam/sdks/go/examples/cookbook/filter'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/forest github.com/apache/beam/sdks/go/examples/forest'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/grades github.com/apache/beam/sdks/go/examples/grades'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/join github.com/apache/beam/sdks/go/examples/cookbook/join'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/max github.com/apache/beam/sdks/go/examples/cookbook/max'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/minimal_wordcount github.com/apache/beam/sdks/go/examples/minimal_wordcount'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/pingpong github.com/apache/beam/sdks/go/examples/pingpong'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/tornadoes github.com/apache/beam/sdks/go/examples/cookbook/tornadoes'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/wordcap github.com/apache/beam/sdks/go/examples/wordcap'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/wordcount github.com/apache/beam/sdks/go/examples/wordcount'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/yatzy github.com/apache/beam/sdks/go/examples/yatzy'
+    go 'build -o ./build/bin/beamctl github.com/apache/beam/sdks/go/cmd/beamctl'
   }
 }
diff --git a/sdks/go/container/build.gradle b/sdks/go/container/build.gradle
@@ -46,13 +46,7 @@ golang {
 }
 
 docker {
-  String repositoryRoot
-  if (rootProject.hasProperty(["docker-repository-root"])) {
-    repositoryRoot = rootProject["docker-repository-root"]
-  } else {
-    repositoryRoot = "${System.properties["user.name"]}-docker-apache.bintray.io/beam"
-  }
-  name "${repositoryRoot}/go:latest"
+  name containerImageName(name: "go")
   files "./build/"
 }
 // Ensure that making the docker image builds any required artifacts