Skip to content
Permalink
Browse files
[FLINK-16880] Add tools and template Dockerfiles to facilitate releas…
…e workflow

This closes #1.
  • Loading branch information
tzulitai committed Apr 7, 2020
1 parent 02a8c7f commit 3ee38250b7dd803e51fe1698f387f82cb3e5df0a
Showing 8 changed files with 407 additions and 0 deletions.
@@ -5,6 +5,32 @@ This repo contains Dockerfiles for building Docker images for [Apache Flink Stat

These Dockerfiles are maintained by the Apache Flink community.

Workflow for new releases
-------------------------

### Release workflow

When a new release of Flink Stateful Functions is available, the Dockerfiles in this repo should be updated.

Updating the Dockerfiles involves 3 steps:

1. Add the GPG key ID of the key used to sign the new release to the `gpg_keys.txt` file.
* Entry format: <statefun_version>=<signing_key>, e.g., 2.0.0=1C1E2394D3194E1944613488F320986D35C33D6A
* Commit this change with message `[release] Add GPG key for x.y.z release`.
2. Run `add-version.sh` with the appropriate arguments (`-s statefun-version -f flink-version`)
* e.g. `./add-version.sh -s 2.0.0 -f 1.10.0`
* Commit these changes with message `[release] Update Dockerfiles for x.y.z release`.

A pull request can then be opened on this repo with the changes.

### Release checklist

- [ ] The GPG key ID of the key used to sign the new release has been added to `gpg_keys.txt` and
committed with the message `[release] Add GPG key for x.y.z release`
- [ ] `./add-version.sh -s x.y.z -f a.b.c` has been run, and the new Dockerfiles committed with
the message `[release] Update Dockerfiles for x.y.z release`
- [ ] A pull request with the above changes has been opened on this repo and merged

License
-------

@@ -0,0 +1,77 @@
#!/bin/bash -e

# Use this script to rebuild the Dockerfiles and all variants for a particular
# release. Before running this, you must first delete the existing release
# directory.
#
# TODO: to conform with other similar setups, this likely needs to become
# "update.sh" and be taught how to derive the latest version (e.g. 1.2.0) from
# a given release (e.g. 1.2) and assemble a .travis.yml file dynamically.
#
# See other repos (e.g. httpd, cassandra) for update.sh examples.

function usage() {
echo >&2 "usage: $0 -s statefun-version -f flink-version"
}

function error() {
local msg="$1"
if [ -n "$2" ]; then
local code="$2"
else
local code=1
fi
echo >&2 "$msg"
exit "$code"
}

statefun_version= # Like 2.0.0
flink_version= # Like 1.10.0

while getopts s:f:h arg; do
case "$arg" in
s)
statefun_version=$OPTARG
;;
f)
flink_version=$OPTARG
;;
h)
usage
exit 0
;;
\?)
usage
exit 1
;;
esac
done

if [ -z "$statefun_version" ] || [ -z "$flink_version" ]; then
usage
exit 1
fi

gpg_key=`grep "$statefun_version" gpg_keys.txt | cut -d '=' -f2`

if [ -z "$gpg_key" ]; then
error "Missing GPG key ID in gpg_keys.txt file for release $statefun_version"
fi

if [ -d "$statefun_version" ]; then
error "Directory $statefun_version already exists; delete before continuing"
fi

echo -n >&2 "Generating Dockerfiles..."
dir="$statefun_version"
mkdir "$dir"

cp -r template/* "$dir"

sed \
-e "s/%%STATEFUN_VERSION%%/$statefun_version/" \
-e "s/%%FLINK_VERSION%%/$flink_version/" \
-e "s/%%GPG_KEY%%/$gpg_key/" \
"template/Dockerfile" > "$dir/Dockerfile"

echo >&2 " done."
Empty file.
@@ -0,0 +1,72 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM flink:%%FLINK_VERSION%%

ENV STATEFUN_VERSION=%%STATEFUN_VERSION%% \
GPG_KEY=%%GPG_KEY%%

ENV ROLE worker
ENV MASTER_HOST localhost
ENV STATEFUN_HOME /opt/statefun
ENV STATEFUN_MODULES $STATEFUN_HOME/modules

# Cleanup flink-lib
RUN rm -fr $FLINK_HOME/lib/flink-table*jar

# Copy our distriubtion template
COPY flink-distribution/ $FLINK_HOME/

# Install Stateful Functions dependencies in Flink lib
ENV DIST_JAR_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-distribution/${STATEFUN_VERSION}/statefun-flink-distribution-${STATEFUN_VERSION}.jar \
DIST_ASC_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-distribution/${STATEFUN_VERSION}/statefun-flink-distribution-${STATEFUN_VERSION}.jar.asc \
CORE_JAR_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-core/${STATEFUN_VERSION}/statefun-flink-core-${STATEFUN_VERSION}.jar \
CORE_ASC_URL=https://repo.maven.apache.org/maven2/org/apache/flink/statefun-flink-core/${STATEFUN_VERSION}/statefun-flink-core-${STATEFUN_VERSION}.jar.asc

RUN set -ex; \
wget -nv -O statefun-flink-distribution.jar "$DIST_JAR_URL"; \
wget -nv -O statefun-flink-distribution.jar.asc "$DIST_ASC_URL"; \
wget -nv -O statefun-flink-core.jar "$CORE_JAR_URL"; \
wget -nv -O statefun-flink-core.jar.asc "$CORE_ASC_URL"; \
\
export GNUPGHOME="$(mktemp -d)"; \
for server in ha.pool.sks-keyservers.net $(shuf -e \
hkp://p80.pool.sks-keyservers.net:80 \
keyserver.ubuntu.com \
hkp://keyserver.ubuntu.com:80 \
pgp.mit.edu) ; do \
gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
done && \
gpg --batch --verify statefun-flink-distribution.jar.asc statefun-flink-distribution.jar; \
gpg --batch --verify statefun-flink-core.jar.asc statefun-flink-core.jar; \
gpgconf --kill all; \
rm -rf "$GNUPGHOME" statefun-flink-distribution.jar.asc statefun-flink-core.jar.asc; \
\
mkdir -p $FLINK_HOME/lib; \
mv statefun-flink-distribution.jar $FLINK_HOME/lib; \
mv statefun-flink-core.jar $FLINK_HOME/lib;

# add user modules
USER root

RUN mkdir -p $STATEFUN_MODULES && \
useradd --system --home-dir $STATEFUN_HOME --uid=9998 --gid=flink statefun && \
chown -R statefun:flink $STATEFUN_HOME && \
chmod -R g+rw $STATEFUN_HOME

# entry point
ADD docker-entry-point.sh /docker-entry-point.sh

ENTRYPOINT ["/docker-entry-point.sh"]
@@ -0,0 +1,56 @@
#!/bin/bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


#
# Role types
#
WORKER="worker"
MASTER="master"

#
# Environment
#
FLINK_HOME=${FLINK_HOME:-"/opt/flink/bin"}
ROLE=${ROLE:-"worker"}
MASTER_HOST=${MASTER_HOST:-"localhost"}

#
# Start a service depending on the role.
#
if [[ "${ROLE}" == "${WORKER}" ]]; then
#
# start the TaskManager (worker role)
#
exec ${FLINK_HOME}/bin/taskmanager.sh start-foreground \
-Djobmanager.rpc.address=${MASTER_HOST}

elif [[ "${ROLE}" == "${MASTER}" ]]; then
#
# start the JobManager (master role) with our predefined job.
#
exec $FLINK_HOME/bin/standalone-job.sh \
start-foreground \
-Djobmanager.rpc.address=${MASTER_HOST} \
"$@"
else
#
# unknown role
#
echo "unknown role ${ROLE}"
exit 1
fi
@@ -0,0 +1,78 @@
#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# This file was taken from Apache Flink, and modified to include another entry point

# Start a Flink service as a console application. Must be stopped with Ctrl-C
# or with SIGTERM by kill or the controlling process.
USAGE="Usage: flink-console.sh (taskexecutor|zookeeper|historyserver|standalonesession|standalonejob|statefun) [args]"

SERVICE=$1
ARGS=("${@:2}") # get remaining arguments as array

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/config.sh

case ${SERVICE} in
(taskexecutor)
CLASS_TO_RUN=org.apache.flink.runtime.taskexecutor.TaskManagerRunner
;;

(historyserver)
CLASS_TO_RUN=org.apache.flink.runtime.webmonitor.history.HistoryServer
;;

(zookeeper)
CLASS_TO_RUN=org.apache.flink.runtime.zookeeper.FlinkZooKeeperQuorumPeer
;;

(standalonesession)
CLASS_TO_RUN=org.apache.flink.runtime.entrypoint.StandaloneSessionClusterEntrypoint
;;

(standalonejob)
CLASS_TO_RUN=org.apache.flink.container.entrypoint.StandaloneJobClusterEntryPoint
;;

(statefun)
CLASS_TO_RUN=org.apache.flink.statefun.flink.launcher.StatefulFunctionsClusterEntryPoint
;;

(*)
echo "Unknown service '${SERVICE}'. $USAGE."
exit 1
;;
esac

FLINK_TM_CLASSPATH=`constructFlinkClassPath`

log_setting=("-Dlog4j.configuration=file:${FLINK_CONF_DIR}/log4j-console.properties" "-Dlogback.configurationFile=file:${FLINK_CONF_DIR}/logback-console.xml")

JAVA_VERSION=$(${JAVA_RUN} -version 2>&1 | sed 's/.*version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')

# Only set JVM 8 arguments if we have correctly extracted the version
if [[ ${JAVA_VERSION} =~ ${IS_NUMBER} ]]; then
if [ "$JAVA_VERSION" -lt 18 ]; then
JVM_ARGS="$JVM_ARGS -XX:MaxPermSize=256m"
fi
fi

echo "Starting $SERVICE as a console application on host $HOSTNAME."
exec $JAVA_RUN ${JVM_ARGS} ${FLINK_ENV_JAVA_OPTS} "${log_setting[@]}" -classpath "`manglePathList "$FLINK_TM_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS"`" ${CLASS_TO_RUN} "${ARGS[@]}"
@@ -0,0 +1,65 @@
#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# This file was taken from Apache Flink, and modified to include another entry point

# Start/stop a Flink JobManager.
USAGE="Usage: standalone-job.sh ((start|start-foreground))|stop [args]"

STARTSTOP=$1
ENTRY_POINT_NAME="statefun"

if [[ ${STARTSTOP} != "start" ]] && [[ ${STARTSTOP} != "start-foreground" ]] && [[ ${STARTSTOP} != "stop" ]]; then
echo ${USAGE}
exit 1
fi

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/config.sh

# Startup parameters
ARGS=("--configDir" "${FLINK_CONF_DIR}" "${@:2}")

if [[ ${STARTSTOP} == "start" ]] || [[ ${STARTSTOP} == "start-foreground" ]]; then
if [ ! -z "${FLINK_JM_HEAP_MB}" ] && [ "${FLINK_JM_HEAP}" == 0 ]; then
echo "used deprecated key \`${KEY_JOBM_MEM_MB}\`, please replace with key \`${KEY_JOBM_MEM_SIZE}\`"
else
flink_jm_heap_bytes=$(parseBytes ${FLINK_JM_HEAP})
FLINK_JM_HEAP_MB=$(getMebiBytes ${flink_jm_heap_bytes})
fi

if [[ ! ${FLINK_JM_HEAP_MB} =~ $IS_NUMBER ]] || [[ "${FLINK_JM_HEAP_MB}" -lt "0" ]]; then
echo "[ERROR] Configured memory size is not a valid value. Please set '${KEY_JOBM_MEM_SIZE}' in ${FLINK_CONF_FILE}."
exit 1
fi

if [ "${FLINK_JM_HEAP_MB}" -gt "0" ]; then
export JVM_ARGS="$JVM_ARGS -Xms"$FLINK_JM_HEAP_MB"m -Xmx"$FLINK_JM_HEAP_MB"m"
fi

# Add cluster entry point specific JVM options
export FLINK_ENV_JAVA_OPTS="${FLINK_ENV_JAVA_OPTS} ${FLINK_ENV_JAVA_OPTS_JM}"
fi

if [[ $STARTSTOP == "start-foreground" ]]; then
exec "${FLINK_BIN_DIR}"/flink-console.sh ${ENTRY_POINT_NAME} "${ARGS[@]}"
else
"${FLINK_BIN_DIR}"/flink-daemon.sh ${STARTSTOP} ${ENTRY_POINT_NAME} "${ARGS[@]}"
fi

0 comments on commit 3ee3825

Please sign in to comment.