[FLINK-16260] Add support for Java11 in dev-master / introduce new release.metadata file

Dockerfile-debian.template

@@ -16,7 +16,7 @@
 # limitations under the License.
 ###############################################################################
 
-FROM openjdk:8-jre
+FROM %%FROM_IMAGE%%
 
 # Install dependencies
 RUN set -ex; \

add-custom.sh

@@ -4,23 +4,27 @@
 # Flink distribution.
 # This is exlusively for development purposes.
 
-source "$(dirname "$0")"/common.sh
+source "$(dirname "$0")"/generator.sh
 
 function usage() {
-    echo >&2 "usage: $0 -u binary-download-url [-n name]"
+    echo >&2 "usage: $0 -u binary-download-url [-n name] [-j java_version]"
 }
 
 binary_download_url=
 name=custom
+java_version=8
 
-while getopts u:n:h arg; do
+while getopts u:n:j:h arg; do
   case "$arg" in
     u)
       binary_download_url=$OPTARG
       ;;
     n)
       name=$OPTARG
       ;;
+    j)
+      java_version=$OPTARG
+      ;;
     h)
       usage
       exit 0
@@ -43,7 +47,7 @@ echo -n >&2 "Generating Dockerfiles..."
 for source_variant in "${SOURCE_VARIANTS[@]}"; do
   dir="dev/${name}-${source_variant}"
   rm -rf "${dir}"
-
-  generate "${dir}" "${binary_download_url}" "" "" false ${source_variant}
+  mkdir "$dir"
+  generateDockerfile "${dir}" "${binary_download_url}" "" "" false ${java_version} ${source_variant}
 done
 echo >&2 " done."

add-version.sh

@@ -10,7 +10,6 @@
 #
 # See other repos (e.g. httpd, cassandra) for update.sh examples.
 
-source "$(dirname "$0")"/common.sh
 
 function usage() {
     echo >&2 "usage: $0 -r flink-release -f flink-version"
@@ -60,6 +59,7 @@ fi
 
 # Defaults, can vary between versions
 scala_versions=( 2.11 2.12 )
+java_versions=( 8 11 )
 gpg_key=
 
 # Version-specific variants (example)
@@ -98,18 +98,24 @@ fi
 
 mkdir "$flink_release"
 
+source "$(dirname "$0")"/generator.sh
+
 echo -n >&2 "Generating Dockerfiles..."
 for source_variant in "${SOURCE_VARIANTS[@]}"; do
     for scala_version in "${scala_versions[@]}"; do
-        dir="$flink_release/scala_${scala_version}-${source_variant}"
+        for java_version in "${java_versions[@]}"; do
+            dir="$flink_release/scala_${scala_version}-java${java_version}-${source_variant}"
 
-        flink_url_file_path=flink/flink-${flink_version}/flink-${flink_version}-bin-scala_${scala_version}.tgz
+            flink_url_file_path=flink/flink-${flink_version}/flink-${flink_version}-bin-scala_${scala_version}.tgz
 
-        flink_tgz_url="https://www.apache.org/dyn/closer.cgi?action=download&filename=${flink_url_file_path}"
-        # Not all mirrors have the .asc files
-        flink_asc_url=https://www.apache.org/dist/${flink_url_file_path}.asc
+            flink_tgz_url="https://www.apache.org/dyn/closer.cgi?action=download&filename=${flink_url_file_path}"
+            # Not all mirrors have the .asc files
+            flink_asc_url=https://www.apache.org/dist/${flink_url_file_path}.asc
 
-        generate "${dir}" "${flink_tgz_url}" "${flink_asc_url}" ${gpg_key} true ${source_variant}
+            mkdir "$dir"
+            generateDockerfile "${dir}" "${flink_tgz_url}" "${flink_asc_url}" ${gpg_key} true ${java_version} ${source_variant}
+            generateReleaseMetadata "${dir}" ${flink_release} ${flink_version} ${scala_version} ${java_version} ${source_variant}
+        done
     done
 done
 echo >&2 " done."

generator.sh

@@ -0,0 +1,90 @@
+#!/bin/bash -e
+
+export SOURCE_VARIANTS=(debian )
+
+export DEFAULT_SCALA="2.12"
+export DEFAULT_JAVA="8"
+
+function generateDockerfile {
+    # define variables
+    dir=$1
+    binary_download_url=$2
+    asc_download_url=$3
+    gpg_key=$4
+    check_gpg=$5
+    java_version=$6
+    source_variant=$7
+
+    from_docker_image="openjdk:${java_version}-jre"
+
+    cp docker-entrypoint.sh "$dir/docker-entrypoint.sh"
+
+    # '&' has special semantics in sed replacement patterns
+    escaped_binary_download_url=$(echo "$binary_download_url" | sed 's/&/\\\&/')
+
+    # generate Dockerfile
+    sed \
+        -e "s,%%BINARY_DOWNLOAD_URL%%,${escaped_binary_download_url}," \
+        -e "s,%%ASC_DOWNLOAD_URL%%,$asc_download_url," \
+        -e "s/%%GPG_KEY%%/$gpg_key/" \
+        -e "s/%%CHECK_GPG%%/${check_gpg}/" \
+        -e "s/%%FROM_IMAGE%%/${from_docker_image}/" \
+        "Dockerfile-$source_variant.template" > "$dir/Dockerfile"
+}
+
+function generateReleaseMetadata {
+    dir=$1
+    flink_release=$2
+    flink_version=$3
+    scala_version=$4
+    java_version=$5
+
+    # docker image tags:
+    java_suffix="-java${java_version}"
+
+    # example "1.2.0-scala_2.11-java11"
+    full_tag=${flink_version}-scala_${scala_version}${java_suffix}
+
+    # example "1.2-scala_2.11-java11"
+    short_tag=${flink_release}-scala_${scala_version}${java_suffix}
+
+    # example "scala_2.12-java11"
+    scala_tag="scala_${scala_version}${java_suffix}"
+
+    tags="$full_tag, $short_tag, $scala_tag"
+
+    if [[ "$java_version" == "$DEFAULT_JAVA" ]]; then
+        # example "1.2.0-scala_2.11"
+        full_tag=${flink_version}-scala_${scala_version}
+
+        # example "1.2-scala_2.11"
+        short_tag=${flink_release}-scala_${scala_version}
+
+        # example "scala_2.12"
+        scala_tag="scala_${scala_version}"
+
+        tags="$tags, $full_tag, $short_tag, $scala_tag"
+    fi
+
+
+    if [[ "$scala_version" == "$DEFAULT_SCALA" ]]; then
+        # we are generating the image for the latest scala version, add:
+        # "1.2.0-java11"
+        # "1.2-java11"
+        # "latest-java11"
+        tags="$tags, ${flink_version}${java_suffix}, ${flink_release}${java_suffix}, latest${java_suffix}"
+
+        if [[ "$java_version" == "$DEFAULT_JAVA" ]]; then
+            # we are generating the image for the default java version, add tags w/o java tag:
+            # "1.2.0"
+            # "1.2"
+            # "latest"
+            tags="$tags, ${flink_version}, ${flink_release}, latest"
+        fi
+    fi
+
+    echo "Tags: $tags" >> $dir/release.metadata
+
+    # We currently only support amd64 with Flink.
+    echo "Architectures: amd64" >> $dir/release.metadata
+}

testing/run_travis_tests.sh

@@ -11,14 +11,15 @@ fi
 
 BRANCH="$TRAVIS_BRANCH"
 
-./add-custom.sh -u "https://s3.amazonaws.com/flink-nightly/flink-1.11-SNAPSHOT-bin-hadoop2.tgz"
-
-if [ -z "$IS_PULL_REQUEST" ] && [ "$BRANCH" = "dev-master" ]; then
-  smoke_test_all_images
-  smoke_test_one_image_non_root
-else
-  # For pull requests and branches, test one image
-  smoke_test_one_image
-fi
+./add-custom.sh -u "https://s3.amazonaws.com/flink-nightly/flink-1.11-SNAPSHOT-bin-hadoop2.tgz" -n test-java8
+
+# test Flink with Java11 image as well
+./add-custom.sh -u "https://s3.amazonaws.com/flink-nightly/flink-1.11-SNAPSHOT-bin-hadoop2.tgz" -j 11 -n test-java11
+
+smoke_test_all_images
+smoke_test_one_image_non_root
+
+
+echo "Test successfully finished"
 
 # vim: et ts=2 sw=2