BIGTOP-2345 Create Flink packaging

bigtop-packages/src/common/flink/do-component-build

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+
+#load versions
+. `dirname $0`/bigtop.bom
+
+# Use Maven to build Flink from source
+mvn install $FLINK_BUILD_OPTS -Drat.skip=true -DskipTests -Dhadoop.version=$HADOOP_VERSION "$@"

bigtop-packages/src/common/flink/flink-jobmanager.svc

@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TYPE="jobmanager"
+DAEMON="flink-${TYPE}"
+DESC="Flink ${TYPE}"
+EXEC_PATH="/usr/lib/flink/bin/flink-daemon.sh"
+WORKING_DIR="/var/lib/flink"
+DAEMON_FLAGS=jobmanager
+CONF_DIR="/etc/flink/conf"
+SVC_USER="flink"
+PIDFILE="/var/run/flink/flink-$SVC_USER-jobmanager.pid"
+EXEC_MODE="cluster"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+    [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+    log_success_msg "Starting $DESC (${DAEMON}): "
+
+    export FLINK_LOG_DIR="/var/log/flink"
+    export FLINK_PID_DIR="/var/run/flink"
+    mkdir -p $FLINK_PID_DIR
+
+    checkstatusofproc
+    status=$?
+    if [ "$status" -eq "$STATUS_RUNNING" ]; then
+        log_success_msg "${DESC} is running"
+        exit 0
+    fi
+
+    LOG_FILE=/var/log/flink/${DAEMON}.out
+    # Flink will set the pid file
+    su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
+    ${EXEC_PATH} start ${DAEMON_FLAGS} --configDir $CONF_DIR --executionMode cluster \
+    > $LOG_FILE 2>&1 & "
+
+    sleep 3
+
+    checkstatusofproc
+
+    RETVAL=$?
+
+    if [ $RETVAL -eq $STATUS_RUNNING ]; then
+    touch $LOCKFILE
+    log_success_msg "Started ${DESC} (${DAEMON}): "
+    else
+    log_failure_msg "Failed to start ${DESC}. Return value: $RETVAL"
+    fi
+    return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+    log_success_msg "Stopping $DESC (${DAEMON}): "
+    killproc -p $PIDFILE java
+    RETVAL=$?
+
+    [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
+    return $RETVAL
+}
+__EOT__
+
+}

bigtop-packages/src/common/flink/flink-taskmanager.svc

@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TYPE="taskmanager"
+DAEMON="flink-${TYPE}"
+DESC="Flink ${TYPE}"
+EXEC_PATH="/usr/lib/flink/bin/flink-daemon.sh"
+WORKING_DIR="/var/lib/flink"
+DAEMON_FLAGS="taskmanager"
+SVC_USER="flink"
+CONF_DIR="/etc/flink/conf"
+PIDFILE="/var/run/flink/flink-$SVC_USER-taskmanager.pid"
+
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+    [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+    log_success_msg "Starting $DESC (${DAEMON}): "
+
+    export FLINK_LOG_DIR="/var/log/flink"
+    export FLINK_PID_DIR="/var/run/flink"
+    mkdir -p $FLINK_PID_DIR
+
+    checkstatusofproc
+    status=$?
+    if [ "$status" -eq "$STATUS_RUNNING" ]; then
+        log_success_msg "${DESC} is running"
+        exit 0
+    fi
+
+    LOG_FILE=/var/log/flink/${DAEMON}.out
+
+    su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
+        ${EXEC_PATH} start ${DAEMON_FLAGS} --configDir ${CONF_DIR} \
+        > $LOG_FILE 2>&1 & "
+
+    sleep 3
+
+    checkstatusofproc
+    RETVAL=$?
+
+    if [ $RETVAL -eq $STATUS_RUNNING ]; then
+     touch $LOCKFILE
+     log_success_msg "Started ${DESC} (${DAEMON}): "
+    else
+     log_failure_msg "Failed to start ${DESC}. Return value: $RETVAL"
+    fi
+
+    return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+    log_success_msg "Stopping $DESC (${DAEMON}): "
+    killproc -p $PIDFILE java
+    RETVAL=$?
+
+    [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
+    return $RETVAL
+}
+__EOT__
+
+}

bigtop-packages/src/common/flink/install_flink.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+  echo "
+usage: $0 <options>
+  Required not-so-options:
+     --build-dir=DIR             path to flink dist.dir
+     --source-dir=DIR            path to package shared files dir
+     --prefix=PREFIX             path to install into
+
+  Optional options:
+     --lib-dir=DIR               path to install flink home [/usr/lib/flink]
+     --installed-lib-dir=DIR     path where lib-dir will end up on target system
+     --bin-dir=DIR               path to install bins [/usr/bin]
+     ... [ see source for more similar options ]
+  "
+  exit 1
+}
+
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'prefix:' \
+  -l 'lib-dir:' \
+  -l 'installed-lib-dir:' \
+  -l 'bin-dir:' \
+  -l 'source-dir:' \
+  -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+    usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+    case "$1" in
+        --prefix)
+        PREFIX=$2 ; shift 2
+        ;;
+        --build-dir)
+        BUILD_DIR=$2 ; shift 2
+        ;;
+        --source-dir)
+        SOURCE_DIR=$2 ; shift 2
+        ;;
+        --lib-dir)
+        LIB_DIR=$2 ; shift 2
+        ;;
+        --installed-lib-dir)
+        INSTALLED_LIB_DIR=$2 ; shift 2
+        ;;
+        --bin-dir)
+        BIN_DIR=$2 ; shift 2
+        ;;
+        --)
+        shift ; break
+        ;;
+        *)
+        echo "Unknown option: $1"
+        usage
+        exit 1
+        ;;
+    esac
+done
+
+for var in PREFIX BUILD_DIR SOURCE_DIR ; do
+  if [ -z "$(eval "echo \$$var")" ]; then
+    echo Missing param: $var
+    usage
+  fi
+done
+
+# load bigtop component versions
+if [ -f "$SOURCE_DIR/bigtop.bom" ]; then
+  . $SOURCE_DIR/bigtop.bom
+fi
+
+
+LIB_DIR=${LIB_DIR:-/usr/lib/flink}
+INSTALLED_LIB_DIR=${INSTALLED_LIB_DIR:-/usr/lib/flink}
+BIN_DIR=${BIN_DIR:-/usr/bin}
+CONF_DIR=${CONF_DIR:-/etc/flink/conf.dist}
+
+install -d -m 0755 $PREFIX/$LIB_DIR
+install -d -m 0755 $PREFIX/$LIB_DIR/bin
+install -d -m 0755 $PREFIX/$LIB_DIR/lib
+install -d -m 0755 $PREFIX/$LIB_DIR/examples
+install -d -m 0755 $PREFIX/$LIB_DIR/resources
+install -d -m 0755 $PREFIX/$CONF_DIR
+install -d -m 0755 $PREFIX/var/log/flink
+install -d -m 0755 $PREFIX/var/log/flink-cli
+install -d -m 0755 $PREFIX/var/run/flink
+
+cp -ra ${BUILD_DIR}/lib/* $PREFIX/${LIB_DIR}/lib/
+cp -a ${BUILD_DIR}/bin/* $PREFIX/${LIB_DIR}/bin/
+# delete Windows start scripts
+rm -rf $PREFIX/${LIB_DIR}/bin/*.cmd
+# remove log directory
+rm -rf  $PREFIX/${LIB_DIR}/log
+
+# Copy the configuration files
+cp -a ${BUILD_DIR}/conf/* $PREFIX/$CONF_DIR
+ln -s /etc/flink/conf $PREFIX/$LIB_DIR/conf
+
+cp -ra ${BUILD_DIR}/examples/* $PREFIX/${LIB_DIR}/examples/
+cp -ra ${BUILD_DIR}/resources/* $PREFIX/${LIB_DIR}/resources/
+
+cp ${BUILD_DIR}/{LICENSE,NOTICE,README.txt} ${PREFIX}/${LIB_DIR}/
+
+# Copy in the /usr/bin/flink wrapper
+install -d -m 0755 $PREFIX/$BIN_DIR
+cat > $PREFIX/$BIN_DIR/flink <<EOF
+#!/bin/bash
+
+# Autodetect JAVA_HOME if not defined
+. /usr/lib/bigtop-utils/bigtop-detect-javahome
+
+export HADOOP_HOME=\${HADOOP_HOME:-/usr/lib/hadoop}
+export HADOOP_CONF_DIR=\${HADOOP_CONF_DIR:-/etc/hadoop/conf}
+export FLINK_HOME=\${FLINK_HOME:-$INSTALLED_LIB_DIR}
+export FLINK_CONF_DIR=\${FLINK_CONF_DIR:-$CONF_DIR}
+export FLINK_LOG_DIR=\${FLINK_LOG_DIR:-/var/log/flink-cli}
+
+exec $INSTALLED_LIB_DIR/bin/flink "\$@"
+EOF
+chmod 755 $PREFIX/$BIN_DIR/flink