From d1d6600cacf247567cfe49245c5bfd1ced41ca60 Mon Sep 17 00:00:00 2001 From: Faraaz Sareshwala Date: Wed, 13 Jan 2016 15:29:46 -0800 Subject: [PATCH] BIGTOP-2285. Add qfs rpm and debian packaging code --- MAINTAINERS.txt | 1 + .../src/common/qfs/do-component-build | 133 +++++++ bigtop-packages/src/common/qfs/install_qfs.sh | 237 ++++++++++++ .../src/common/qfs/qfs-chunkserver.svc | 89 +++++ .../src/common/qfs/qfs-metaserver.svc | 89 +++++ bigtop-packages/src/common/qfs/qfs-webui.svc | 89 +++++ bigtop-packages/src/deb/qfs/changelog | 1 + bigtop-packages/src/deb/qfs/compat | 1 + bigtop-packages/src/deb/qfs/control | 99 +++++ bigtop-packages/src/deb/qfs/copyright | 15 + .../src/deb/qfs/qfs-chunkserver.dirs | 2 + .../src/deb/qfs/qfs-chunkserver.install | 3 + .../deb/qfs/qfs-chunkserver.lintian-overrides | 2 + .../src/deb/qfs/qfs-chunkserver.postinst | 35 ++ .../src/deb/qfs/qfs-chunkserver.postrm | 32 ++ .../src/deb/qfs/qfs-chunkserver.preinst | 34 ++ .../src/deb/qfs/qfs-client.install | 33 ++ .../src/deb/qfs/qfs-client.lintian-overrides | 3 + bigtop-packages/src/deb/qfs/qfs-dev.install | 27 ++ .../src/deb/qfs/qfs-dev.lintian-overrides | 2 + bigtop-packages/src/deb/qfs/qfs-fuse.install | 1 + .../src/deb/qfs/qfs-fuse.lintian-overrides | 2 + .../src/deb/qfs/qfs-hadoop.install | 1 + .../src/deb/qfs/qfs-hadoop.lintian-overrides | 2 + bigtop-packages/src/deb/qfs/qfs-java.install | 1 + .../src/deb/qfs/qfs-java.lintian-overrides | 1 + .../src/deb/qfs/qfs-metaserver.dirs | 2 + .../src/deb/qfs/qfs-metaserver.install | 4 + .../deb/qfs/qfs-metaserver.lintian-overrides | 2 + .../src/deb/qfs/qfs-metaserver.postinst | 35 ++ .../src/deb/qfs/qfs-metaserver.postrm | 32 ++ .../src/deb/qfs/qfs-metaserver.preinst | 34 ++ .../src/deb/qfs/qfs-python.install | 1 + .../src/deb/qfs/qfs-python.lintian-overrides | 1 + bigtop-packages/src/deb/qfs/qfs-webui.dirs | 2 + bigtop-packages/src/deb/qfs/qfs-webui.install | 2 + .../src/deb/qfs/qfs-webui.lintian-overrides | 1 + .../src/deb/qfs/qfs-webui.postinst | 35 ++ bigtop-packages/src/deb/qfs/qfs-webui.postrm | 32 ++ bigtop-packages/src/deb/qfs/qfs-webui.preinst | 34 ++ .../src/deb/qfs/qfs.lintian-overrides | 2 + bigtop-packages/src/deb/qfs/rules | 81 ++++ bigtop-packages/src/deb/qfs/source/format | 1 + bigtop-packages/src/rpm/qfs/SPECS/qfs.spec | 360 ++++++++++++++++++ bigtop-packages/src/templates/init.d.tmpl | 8 +- bigtop.bom | 9 + bigtop_toolchain/manifests/packages.pp | 6 +- packages.gradle | 4 + 48 files changed, 1617 insertions(+), 6 deletions(-) create mode 100644 bigtop-packages/src/common/qfs/do-component-build create mode 100644 bigtop-packages/src/common/qfs/install_qfs.sh create mode 100644 bigtop-packages/src/common/qfs/qfs-chunkserver.svc create mode 100644 bigtop-packages/src/common/qfs/qfs-metaserver.svc create mode 100644 bigtop-packages/src/common/qfs/qfs-webui.svc create mode 100644 bigtop-packages/src/deb/qfs/changelog create mode 100644 bigtop-packages/src/deb/qfs/compat create mode 100644 bigtop-packages/src/deb/qfs/control create mode 100644 bigtop-packages/src/deb/qfs/copyright create mode 100644 bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs create mode 100644 bigtop-packages/src/deb/qfs/qfs-chunkserver.install create mode 100644 bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides create mode 100644 bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst create mode 100644 bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm create mode 100644 bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst create mode 100644 bigtop-packages/src/deb/qfs/qfs-client.install create mode 100644 bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides create mode 100644 bigtop-packages/src/deb/qfs/qfs-dev.install create mode 100644 bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides create mode 100644 bigtop-packages/src/deb/qfs/qfs-fuse.install create mode 100644 bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides create mode 100644 bigtop-packages/src/deb/qfs/qfs-hadoop.install create mode 100644 bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides create mode 100644 bigtop-packages/src/deb/qfs/qfs-java.install create mode 100644 bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides create mode 100644 bigtop-packages/src/deb/qfs/qfs-metaserver.dirs create mode 100644 bigtop-packages/src/deb/qfs/qfs-metaserver.install create mode 100644 bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides create mode 100644 bigtop-packages/src/deb/qfs/qfs-metaserver.postinst create mode 100644 bigtop-packages/src/deb/qfs/qfs-metaserver.postrm create mode 100644 bigtop-packages/src/deb/qfs/qfs-metaserver.preinst create mode 100644 bigtop-packages/src/deb/qfs/qfs-python.install create mode 100644 bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides create mode 100644 bigtop-packages/src/deb/qfs/qfs-webui.dirs create mode 100644 bigtop-packages/src/deb/qfs/qfs-webui.install create mode 100644 bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides create mode 100644 bigtop-packages/src/deb/qfs/qfs-webui.postinst create mode 100644 bigtop-packages/src/deb/qfs/qfs-webui.postrm create mode 100644 bigtop-packages/src/deb/qfs/qfs-webui.preinst create mode 100644 bigtop-packages/src/deb/qfs/qfs.lintian-overrides create mode 100755 bigtop-packages/src/deb/qfs/rules create mode 100644 bigtop-packages/src/deb/qfs/source/format create mode 100644 bigtop-packages/src/rpm/qfs/SPECS/qfs.spec diff --git a/MAINTAINERS.txt b/MAINTAINERS.txt index 495480128a..0b1f4a6a0a 100644 --- a/MAINTAINERS.txt +++ b/MAINTAINERS.txt @@ -17,6 +17,7 @@ oozie evans ye, rvs phoenix: andrew purtell, youngwoo kim pig: evans ye, daniel dai puppet recipes: jay vyas, cos, evans ye, rvs +qfs: fsareshwala smoke-tests: jay vyas, david capwell spark: jay vyas, youngwoo kim sqoop: sean mackrory, youngwoo kim diff --git a/bigtop-packages/src/common/qfs/do-component-build b/bigtop-packages/src/common/qfs/do-component-build new file mode 100644 index 0000000000..dd269d0e59 --- /dev/null +++ b/bigtop-packages/src/common/qfs/do-component-build @@ -0,0 +1,133 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For debugging: +# set -ex + +# Autodetect JAVA_HOME if not defined +source /usr/lib/bigtop-utils/bigtop-detect-javahome + +usage() { + echo " +usage: $0 + Required not-so-options: + --qfs-version=VERSION version of qfs we are installing + --hadoop-version=VERSION version of hadoop qfs built against + --python=PATH path to python + + Optional options: + --python3=PATH path to python 3 + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'hadoop-version:' \ + -l 'qfs-version:' \ + -l 'python:' \ + -l 'python3:' \ + -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --hadoop-version) + HADOOP_VERSION=$2 ; shift 2 + ;; + --qfs-version) + QFS_VERSION=$2 ; shift 2 + ;; + --python) + PYTHON_PATH=$2 ; shift 2 + ;; + --python3) + PYTHON3_PATH=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in QFS_VERSION HADOOP_VERSION PYTHON_PATH ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing required param: $var + usage + fi +done + +# bigtop installs maven on its own and not through the package manager +export PATH=/usr/local/maven/bin:$PATH +command -v mvn > /dev/null 2>&1 || { + echo >&2 "Maven is required but not found in PATH. Check your installation." + exit 1 +} + +PYTHON3_PATH=${PYTHON3_PATH:-} + +# Build and install QFS into the build/release directory +make + +# Build QFS Java library JAR and Hadoop plugin JAR. +# NOTE(fsareshwala): we have src/java/javabuild.sh to do this for us, but that +# script does some git magic to determine the release numbers and hashes from +# git tags. However, in bigtop, we have an entirely different set of git objects +# and will get the wrong tag name and release hash if we use javabuild.sh as it +# currently is. Until qfs can update its build process upstream, we replicate +# what javabuild.sh would do. +get_hadoop_qfs_profile() { + myversion="`echo "$HADOOP_VERSION" | cut -d. -f 1-2`" + myversionmaj="`echo "$HADOOP_VERSION" | cut -d. -f 1`" + + if [ x"$myversion" = x"1.0" -o x"$myversion" = x"1.1" ]; then + echo "hadoop_branch1_profile" + elif [ x"$myversion" = x"0.23" ]; then + echo "hadoop_trunk_profile" + elif [ x"$myversionmaj" = x"2" ]; then + echo "hadoop_trunk_profile,hadoop_trunk_profile_2" + else + echo "Unsupported Hadoop release version." + exit 1 + fi +} + +HADOOP_QFS_PROFILE=$(get_hadoop_qfs_profile) +cd src/java +mvn -Dqfs.release.version=${QFS_VERSION} --projects qfs-access package +mvn -P $HADOOP_QFS_PROFILE -Dqfs.release.version=${QFS_VERSION} \ + -Dhadoop.release.version=${HADOOP_VERSION} -Dtest.build.data=/tmp \ + package +cd ../.. + +# Build QFS Python libraries. +cd build/release +$PYTHON_PATH ../../src/cc/access/kfs_setup.py build +if [ ! -z "$PYTHON3_PATH" ]; then + $PYTHON3_PATH ../../src/cc/access/kfs_setup.py build +fi +cd .. diff --git a/bigtop-packages/src/common/qfs/install_qfs.sh b/bigtop-packages/src/common/qfs/install_qfs.sh new file mode 100644 index 0000000000..10ef5376a3 --- /dev/null +++ b/bigtop-packages/src/common/qfs/install_qfs.sh @@ -0,0 +1,237 @@ +#!/bin/bash -x + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +usage() { + echo " +usage: $0 + Required not-so-options: + --prefix=PREFIX path to install into + --qfs-version=VERSION version of qfs we are installing + --hadoop-version=VERSION version of hadoop qfs built against + --python=PYTHON the path to python + + Optional options: + --bin-dir=DIR path to install binaries (default: /usr/bin) + --lib-dir=DIR path to install libraries (default: /usr/lib) + --etc-dir=DIR path to install configuration (default: /etc/qfs) + --include-dir=DIR path to install devel headers (default: /usr/include) + --data-dir=DIR path to install various data (default: /usr/share/qfs) + --var-dir=DIR path to keep variable data (default: /var/qfs) + --hadoop-home=DIR path where hadoop lives (default: /usr/lib/hadoop) + --python3=PYTHON3 path to python 3 + --python-extra=PYTHON_EXTRA extra arguments to pass to python when installing + + Note: you will have to change the init scripts with the new paths you use + above if you deviate from the defaults. + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'bin-dir:' \ + -l 'lib-dir:' \ + -l 'etc-dir:' \ + -l 'include-dir:' \ + -l 'data-dir:' \ + -l 'var-dir:' \ + -l 'hadoop-home:' \ + -l 'qfs-version:' \ + -l 'hadoop-version:' \ + -l 'python:' \ + -l 'python3:' \ + -l 'python-extra:' \ + -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --etc-dir) + ETC_DIR=$2 ; shift 2 + ;; + --include-dir) + INCLUDE_DIR=$2 ; shift 2 + ;; + --data-dir) + DATA_DIR=$2 ; shift 2 + ;; + --var-dir) + VAR_DIR=$2 ; shift 2 + ;; + --hadoop-home) + HADOOP_HOME=$2 ; shift 2 + ;; + --qfs-version) + QFS_VERSION=$2 ; shift 2 + ;; + --hadoop-version) + HADOOP_VERSION=$2 ; shift 2 + ;; + --python) + PYTHON_PATH=$2 ; shift 2 + ;; + --python3) + PYTHON3_PATH=$2 ; shift 2 + ;; + --python-extra) + PYTHON_EXTRA=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX QFS_VERSION HADOOP_VERSION PYTHON_PATH ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing required param: $var + usage + fi +done + +PREFIX=$(readlink -f $PREFIX) +BIN_DIR=$PREFIX/${BIN_DIR:-/usr/bin/qfs} +LIB_DIR=$PREFIX/${LIB_DIR:-/usr/lib/qfs} +ETC_DIR=$PREFIX/${ETC_DIR:-/etc/qfs} +INCLUDE_DIR=$PREFIX/${INCLUDE_DIR:-/usr/include} +DATA_DIR=$PREFIX/${DATA_DIR:-/usr/share/qfs} +VAR_DIR=$PREFIX/${VAR_DIR:-/var/qfs} +HADOOP_HOME=$PREFIX/${HADOOP_HOME:-/usr/lib/hadoop} +PYTHON_EXTRA=${PYTHON_EXTRA:-} + +mkdir -p $BIN_DIR +install -m 755 build/release/bin/filelister $BIN_DIR +install -m 755 build/release/bin/logcompactor $BIN_DIR +install -m 755 build/release/bin/metaserver $BIN_DIR +install -m 755 build/release/bin/qfsfsck $BIN_DIR +install -m 755 build/release/bin/qfs_fuse $BIN_DIR +install -m 755 build/release/bin/chunkserver $BIN_DIR +install -m 755 build/release/bin/chunkscrubber $BIN_DIR + +install -m 755 build/release/bin/devtools/checksum $BIN_DIR +install -m 755 build/release/bin/devtools/dirtree_creator $BIN_DIR +install -m 755 build/release/bin/devtools/dtokentest $BIN_DIR +install -m 755 build/release/bin/devtools/logger $BIN_DIR/qfslogger +install -m 755 build/release/bin/devtools/rand-sfmt $BIN_DIR +install -m 755 build/release/bin/devtools/requestparser $BIN_DIR +install -m 755 build/release/bin/devtools/sortedhash $BIN_DIR +install -m 755 build/release/bin/devtools/sslfiltertest $BIN_DIR +install -m 755 build/release/bin/devtools/stlset $BIN_DIR + +install -m 755 build/release/bin/emulator/rebalanceexecutor $BIN_DIR +install -m 755 build/release/bin/emulator/rebalanceplanner $BIN_DIR +install -m 755 build/release/bin/emulator/replicachecker $BIN_DIR + +install -m 755 build/release/bin/examples/qfssample $BIN_DIR + +install -m 755 build/release/bin/tools/cpfromqfs $BIN_DIR +install -m 755 build/release/bin/tools/cptoqfs $BIN_DIR +install -m 755 build/release/bin/tools/qfs $BIN_DIR +install -m 755 build/release/bin/tools/qfsadmin $BIN_DIR +install -m 755 build/release/bin/tools/qfscat $BIN_DIR +install -m 755 build/release/bin/tools/qfsdataverify $BIN_DIR +install -m 755 build/release/bin/tools/qfsfileenum $BIN_DIR +install -m 755 build/release/bin/tools/qfshibernate $BIN_DIR +install -m 755 build/release/bin/tools/qfsping $BIN_DIR +install -m 755 build/release/bin/tools/qfsput $BIN_DIR +install -m 755 build/release/bin/tools/qfsshell $BIN_DIR +install -m 755 build/release/bin/tools/qfsstats $BIN_DIR +install -m 755 build/release/bin/tools/qfstoggleworm $BIN_DIR + +install -m 755 scripts/qfs_backup $BIN_DIR +install -m 755 scripts/qfs_checkpoint_prune.py $BIN_DIR/qfs_checkpoint_prune +install -m 755 scripts/qfs_log_prune.py $BIN_DIR/qfs_log_prune + +mkdir -p $LIB_DIR +install -m 644 build/release/lib/libgf_complete.so $LIB_DIR +install -m 644 build/release/lib/libgf_complete.so.1 $LIB_DIR +install -m 644 build/release/lib/libgf_complete.so.1.0.0 $LIB_DIR +install -m 644 build/release/lib/libJerasure.so $LIB_DIR +install -m 644 build/release/lib/libJerasure.so.2 $LIB_DIR +install -m 644 build/release/lib/libJerasure.so.2.0.0 $LIB_DIR +install -m 644 build/release/lib/libqfs_access.so $LIB_DIR +install -m 644 build/release/lib/libqfs_client.so $LIB_DIR +install -m 644 build/release/lib/libqfs_common.so $LIB_DIR +install -m 644 build/release/lib/libqfsc.so $LIB_DIR +install -m 644 build/release/lib/libqfs_io.so $LIB_DIR +install -m 644 build/release/lib/libqfskrb.so $LIB_DIR +install -m 644 build/release/lib/libqfs_qcdio.so $LIB_DIR +install -m 644 build/release/lib/libqfs_qcrs.so $LIB_DIR + +install -m 644 build/release/lib/static/libgf_complete.a $LIB_DIR +install -m 644 build/release/lib/static/libJerasure.a $LIB_DIR +install -m 644 build/release/lib/static/libqfsc.a $LIB_DIR +install -m 644 build/release/lib/static/libqfs_client.a $LIB_DIR +install -m 644 build/release/lib/static/libqfs_common.a $LIB_DIR +install -m 644 build/release/lib/static/libqfs_emulator.a $LIB_DIR +install -m 644 build/release/lib/static/libqfs_io.a $LIB_DIR +install -m 644 build/release/lib/static/libqfskrb.a $LIB_DIR +install -m 644 build/release/lib/static/libqfs_meta.a $LIB_DIR +install -m 644 build/release/lib/static/libqfs_qcdio.a $LIB_DIR +install -m 644 build/release/lib/static/libqfs_qcrs.a $LIB_DIR +install -m 644 build/release/lib/static/libqfs_tools.a $LIB_DIR + +mkdir -p $ETC_DIR/logrotate.d +install -m 644 contrib/logrotate/qfs-chunkserver $ETC_DIR/logrotate.d +install -m 644 contrib/logrotate/qfs-metaserver $ETC_DIR/logrotate.d +install -m 644 contrib/logrotate/qfs-webui $ETC_DIR/logrotate.d + +mkdir -p $INCLUDE_DIR +cp -a build/release/include/* $INCLUDE_DIR + +mkdir -p $DATA_DIR/webui +cp -rp webui/* $DATA_DIR/webui + +mkdir -p $DATA_DIR/java +install -m 644 build/java/qfs-access/qfs-access-${QFS_VERSION}.jar $DATA_DIR/java + +mkdir -p $HADOOP_HOME/lib +install -m 644 build/java/hadoop-qfs/hadoop-${HADOOP_VERSION}-qfs-${QFS_VERSION}.jar ${HADOOP_HOME}/lib + +cd build/release +$PYTHON_PATH ../../src/cc/access/kfs_setup.py install --skip-build --root=$PREFIX --prefix=/usr $PYTHON_EXTRA +if [ ! -z "$PYTHON3_PATH" ]; then + $PYTHON3_PATH ../../src/cc/access/kfs_setup.py install --skip-build --root=$PREFIX --prefix=/usr $PYTHON_EXTRA +fi +cd .. + +mkdir -p $VAR_DIR/metaserver/checkpoint +mkdir -p $VAR_DIR/log/qfs +mkdir -p $VAR_DIR/run/qfs +mkdir -p $VAR_DIR/lib/qfs diff --git a/bigtop-packages/src/common/qfs/qfs-chunkserver.svc b/bigtop-packages/src/common/qfs/qfs-chunkserver.svc new file mode 100644 index 0000000000..304d9cfa9c --- /dev/null +++ b/bigtop-packages/src/common/qfs/qfs-chunkserver.svc @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DAEMON="chunkserver" +DESC="QFS Chunkserver" +PROC_NAME="chunkserver" +EXEC_PATH="/usr/bin/${PROC_NAME}" +SVC_USER="qfs" +WORKING_DIR="/var/lib/qfs" +CONF_DIR="/etc/qfs" +CONF_FILE="ChunkServer.prp" +LOGFILE="/var/log/qfs/${DAEMON}.log" +PIDFILE="/var/run/qfs/${DAEMON}.pid" + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED + log_success_msg "Starting $DESC ($DAEMON): " + + checkstatusofproc + status=$? + if [ "$status" -eq "$STATUS_RUNNING" ]; then + log_success_msg "$DESC is running" + exit 0 + fi + + echo "Starting $DESC: " + + mkdir -p $(dirname $PIDFILE) + su -s /bin/bash $SVC_USER -c "cd $WORKING_DIR && nohup $EXEC_PATH $CONF_DIR/$CONF_FILE > $LOGFILE 2>&1 < /dev/null & "'echo $! '"> $PIDFILE" + sleep 3 + + checkstatusofproc + RETVAL=$? + if [ $RETVAL -eq $STATUS_RUNNING ]; then + touch $LOCKFILE + log_success_msg "Starting $DESC (${DAEMON}): " + else + log_failure_msg "Failure to start $DESC (${DAEMON}). Return value: $RETVAL" + fi + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { + checkstatusofproc + if [ "$?" = "$STATUS_RUNNING" ] ; then + if [ -f $PIDFILE ]; then + PID=`cat $PIDFILE` + if [ -n $PID ]; then + kill -TERM $PID &>/dev/null + fi + fi + RETVAL=$? + else + RETVAL=$RETVAL_SUCCESS + fi + + if [ $RETVAL -eq $RETVAL_SUCCESS ]; then + rm -f $LOCKFILE $PIDFILE + log_success_msg "Stopped ${DESC}: " + else + log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL" + fi + return $RETVAL +} + +__EOT__ + +} diff --git a/bigtop-packages/src/common/qfs/qfs-metaserver.svc b/bigtop-packages/src/common/qfs/qfs-metaserver.svc new file mode 100644 index 0000000000..69b8170578 --- /dev/null +++ b/bigtop-packages/src/common/qfs/qfs-metaserver.svc @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DAEMON="metaserver" +DESC="QFS Metaserver" +PROC_NAME="metaserver" +EXEC_PATH="/usr/bin/${PROC_NAME}" +SVC_USER="qfs" +WORKING_DIR="/var/lib/qfs" +CONF_DIR="/etc/qfs" +CONF_FILE="MetaServer.prp" +LOGFILE="/var/log/qfs/${DAEMON}.log" +PIDFILE="/var/run/qfs/${DAEMON}.pid" + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED + log_success_msg "Starting $DESC ($DAEMON): " + + checkstatusofproc + status=$? + if [ "$status" -eq "$STATUS_RUNNING" ]; then + log_success_msg "$DESC is running" + exit 0 + fi + + echo "Starting $DESC: " + + mkdir -p $(dirname $PIDFILE) + su -s /bin/bash $SVC_USER -c "cd $WORKING_DIR && nohup $EXEC_PATH $CONF_DIR/$CONF_FILE > $LOGFILE 2>&1 < /dev/null & "'echo $! '"> $PIDFILE" + sleep 3 + + checkstatusofproc + RETVAL=$? + if [ $RETVAL -eq $STATUS_RUNNING ]; then + touch $LOCKFILE + log_success_msg "Starting $DESC (${DAEMON}): " + else + log_failure_msg "Failure to start $DESC (${DAEMON}). Return value: $RETVAL" + fi + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { + checkstatusofproc + if [ "$?" = "$STATUS_RUNNING" ] ; then + if [ -f $PIDFILE ]; then + PID=`cat $PIDFILE` + if [ -n $PID ]; then + kill -TERM $PID &>/dev/null + fi + fi + RETVAL=$? + else + RETVAL=$RETVAL_SUCCESS + fi + + if [ $RETVAL -eq $RETVAL_SUCCESS ]; then + rm -f $LOCKFILE $PIDFILE + log_success_msg "Stopped ${DESC}: " + else + log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL" + fi + return $RETVAL +} + +__EOT__ + +} diff --git a/bigtop-packages/src/common/qfs/qfs-webui.svc b/bigtop-packages/src/common/qfs/qfs-webui.svc new file mode 100644 index 0000000000..04add6ddf2 --- /dev/null +++ b/bigtop-packages/src/common/qfs/qfs-webui.svc @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DAEMON="webui" +DESC="QFS WebUI" +PROC_NAME="python" +EXEC_PATH="/usr/bin/${PROC_NAME} /usr/share/qfs/webui/qfsstatus.py" +SVC_USER="qfs" +WORKING_DIR="/var/lib/qfs" +CONF_DIR="/usr/share/qfs/webui" +CONF_FILE="server.conf" +LOGFILE="/var/log/qfs/${DAEMON}.log" +PIDFILE="/var/run/qfs/${DAEMON}.pid" + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED + log_success_msg "Starting $DESC ($DAEMON): " + + checkstatusofproc + status=$? + if [ "$status" -eq "$STATUS_RUNNING" ]; then + log_success_msg "$DESC is running" + exit 0 + fi + + echo "Starting $DESC: " + + mkdir -p $(dirname $PIDFILE) + su -s /bin/bash $SVC_USER -c "cd $WORKING_DIR && nohup $EXEC_PATH $CONF_DIR/$CONF_FILE > $LOGFILE 2>&1 < /dev/null & "'echo $! '"> $PIDFILE" + sleep 3 + + checkstatusofproc + RETVAL=$? + if [ $RETVAL -eq $STATUS_RUNNING ]; then + touch $LOCKFILE + log_success_msg "Starting $DESC (${DAEMON}): " + else + log_failure_msg "Failure to start $DESC (${DAEMON}). Return value: $RETVAL" + fi + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { + checkstatusofproc + if [ "$?" = "$STATUS_RUNNING" ] ; then + if [ -f $PIDFILE ]; then + PID=`cat $PIDFILE` + if [ -n $PID ]; then + kill -TERM $PID &>/dev/null + fi + fi + RETVAL=$? + else + RETVAL=$RETVAL_SUCCESS + fi + + if [ $RETVAL -eq $RETVAL_SUCCESS ]; then + rm -f $LOCKFILE $PIDFILE + log_success_msg "Stopped ${DESC}: " + else + log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL" + fi + return $RETVAL +} + +__EOT__ + +} diff --git a/bigtop-packages/src/deb/qfs/changelog b/bigtop-packages/src/deb/qfs/changelog new file mode 100644 index 0000000000..d4858cd496 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/changelog @@ -0,0 +1 @@ +--- This is auto-generated diff --git a/bigtop-packages/src/deb/qfs/compat b/bigtop-packages/src/deb/qfs/compat new file mode 100644 index 0000000000..ec635144f6 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/compat @@ -0,0 +1 @@ +9 diff --git a/bigtop-packages/src/deb/qfs/control b/bigtop-packages/src/deb/qfs/control new file mode 100644 index 0000000000..2bce55da25 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/control @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Source: qfs +Section: misc +Priority: extra +Maintainer: Apache Bigtop +Build-Depends: cmake, libboost-regex-dev, xfslibs-dev, libssl-dev, python2.7-dev | python3-dev, libfuse-dev, dh-python, debhelper (>= 7.0.50~) +Standards-Version: 3.9.4 +Homepage: https://quantcast.github.io/qfs + +Package: qfs +Depends: qfs-client (= ${binary:Version}), qfs-chunkserver (= ${binary:Version}), qfs-metaserver (= ${binary:Version}), qfs-dev (= ${binary:Version}), qfs-fuse (= ${binary:Version}), qfs-hadoop (= ${binary:Version}), qfs-java (= ${binary:Version}), qfs-python (= ${binary:Version}), qfs-webui (= ${binary:Version}) +Architecture: any +Description: Metapackage for all QFS components + Quantcast File System (QFS) is a high-performance, fault-tolerant, distributed + file system developed to support MapReduce processing, or other applications + reading and writing large files sequentially. + +Package: qfs-client +Depends: ${shlibs:Depends}, ${misc:Depends}, python +Architecture: any +Description: Client binaries and libraries used to link against qfs + This package provides base client binaries and libraries used to interact with + a running qfs cluster and link against qfs APIs to build your own software that + interacts with a running qfs cluster. + +Package: qfs-chunkserver +Depends: ${shlibs:Depends}, ${misc:Depends} +Recommends: logrotate +Architecture: any +Description: Executables required to run the Quantcast File System chunkserver + The QFS chunkserver service hosts the binary contents of the QFS distributed + filesystem. A metaserver coordinates many data nodes running this service, + replicating data amongst chunkservers in a redundant fashion. + +Package: qfs-metaserver +Depends: ${shlibs:Depends}, ${misc:Depends} +Recommends: logrotate +Architecture: any +Description: Executables required to run the Quantcast File System metaserver + This package contains the executables required to run the Quantcast File System + metaserver service, which tracks the location of data chunks distributed across + QFS chunkservers. + +Package: qfs-dev +Depends: ${shlibs:Depends}, ${misc:Depends} +Architecture: any +Description: Files needed for building Quantcast File System-based applications + The QFS devel package contains the headers, static libraries, and developer + tool binaries required to develop applications which build against QFS. + +Package: qfs-fuse +Depends: ${shlibs:Depends}, ${misc:Depends} +Architecture: any +Description: Support for mounting the Quantcast File System under FUSE + This package contains the qfs_fuse executable which is required when mounting + QFS distributed filesystems under FUSE. + +Package: qfs-hadoop +Depends: ${misc:Depends} +Architecture: any +Description: Quantcast File System plugin JAR for Hadoop + This package contains a plugin JAR to enable QFS to serve as a drop-in + replacement for HDFS under Hadoop. + +Package: qfs-java +Depends: ${misc:Depends} +Architecture: any +Description: Java libraries for accessing the Quantcast File System + This package contains a JAR which enables Java applications to access QFS via + its JNI interface. + +Package: qfs-python +Depends: ${shlibs:Depends}, ${misc:Depends}, ${python:Depends} +Architecture: any +Description: Python libraries for accessing the Quantcast File System + This package contains the libraries required to access the Quantcast File + System libraries via Python. + +Package: qfs-webui +Depends: ${misc:Depends}, ${misc:Depends}, python +Recommends: logrotate +Architecture: all +Description: Quantcast File System metaserver/chunkserver web frontend + This package contains several Python scripts which provide a simple Hadoop-like + Web UI for viewing Quantcast File Server chunkserver and metaserver status. diff --git a/bigtop-packages/src/deb/qfs/copyright b/bigtop-packages/src/deb/qfs/copyright new file mode 100644 index 0000000000..5d02a4edca --- /dev/null +++ b/bigtop-packages/src/deb/qfs/copyright @@ -0,0 +1,15 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: qfs +Source: https://quantcast.github.io/qfs + +Files: * +Copyright: 2010-2011, The Apache Software Foundation +License: Apache-2.0 + +Files debian/* +Copyright: 2011, The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + On Debian systems, the complete text of the Apache 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs b/bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs new file mode 100644 index 0000000000..725c19aa0c --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs @@ -0,0 +1,2 @@ +/var/log/qfs +/var/lib/qfs diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.install b/bigtop-packages/src/deb/qfs/qfs-chunkserver.install new file mode 100644 index 0000000000..9f9390c1b0 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.install @@ -0,0 +1,3 @@ +/usr/bin/chunkserver +/usr/bin/chunkscrubber +/etc/qfs/logrotate.d/qfs-chunkserver diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides new file mode 100644 index 0000000000..ca75f34f17 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides @@ -0,0 +1,2 @@ +qfs-chunkserver binary: new-package-should-close-itp-bug +qfs-chunkserver binary: binary-without-manpage diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst b/bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst new file mode 100644 index 0000000000..40b5ea5d57 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + configure) + update-rc.d qfs-chunkserver defaults >/dev/null || exit 1 + chown qfs:qfs -R /var/log/qfs + chown qfs:qfs -R /var/run/qfs + chown qfs:qfs -R /var/lib/qfs + ;; + abort-upgrade|abort-remove|abort-deconfigure) + ;; + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm b/bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm new file mode 100644 index 0000000000..d6098a79a3 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + purge) + update-rc.d -f qfs-chunkserver remove > /dev/null || exit 1 + ;; + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst b/bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst new file mode 100644 index 0000000000..7416ecd4a4 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + install|upgrade) + getent group qfs >/dev/null || groupadd -r qfs + getent passwd qfs > /dev/null || useradd -c "qfs" -s /bin/bash -g qfs \ + -d /var/lib/qfs qfs 2> /dev/null || : + ;; + abort-upgrade) + ;; + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# diff --git a/bigtop-packages/src/deb/qfs/qfs-client.install b/bigtop-packages/src/deb/qfs/qfs-client.install new file mode 100644 index 0000000000..225dc0f4b3 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-client.install @@ -0,0 +1,33 @@ +/usr/bin/qfs +/usr/bin/qfsadmin +/usr/bin/qfscat +/usr/bin/qfsdataverify +/usr/bin/qfsfileenum +/usr/bin/qfsfsck +/usr/bin/qfshibernate +/usr/bin/qfsping +/usr/bin/qfsput +/usr/bin/qfsshell +/usr/bin/qfsstats +/usr/bin/qfstoggleworm +/usr/bin/qfssample +/usr/bin/qfs_backup +/usr/bin/qfs_checkpoint_prune +/usr/bin/qfs_log_prune +/usr/bin/cpfromqfs +/usr/bin/cptoqfs + +/usr/lib/qfs/libgf_complete.so +/usr/lib/qfs/libgf_complete.so.1 +/usr/lib/qfs/libgf_complete.so.1.0.0 +/usr/lib/qfs/libJerasure.so +/usr/lib/qfs/libJerasure.so.2 +/usr/lib/qfs/libJerasure.so.2.0.0 +/usr/lib/qfs/libqfs_access.so +/usr/lib/qfs/libqfs_client.so +/usr/lib/qfs/libqfs_common.so +/usr/lib/qfs/libqfsc.so +/usr/lib/qfs/libqfs_io.so +/usr/lib/qfs/libqfskrb.so +/usr/lib/qfs/libqfs_qcdio.so +/usr/lib/qfs/libqfs_qcrs.so diff --git a/bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides new file mode 100644 index 0000000000..a694309368 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides @@ -0,0 +1,3 @@ +qfs-client binary: new-package-should-close-itp-bug +qfs-client binary: binary-or-shlib-defines-rpath +qfs-client binary: binary-without-manpage diff --git a/bigtop-packages/src/deb/qfs/qfs-dev.install b/bigtop-packages/src/deb/qfs/qfs-dev.install new file mode 100644 index 0000000000..8371fa0c6a --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-dev.install @@ -0,0 +1,27 @@ +/usr/bin/checksum +/usr/bin/dirtree_creator +/usr/bin/dtokentest +/usr/bin/qfslogger +/usr/bin/rand-sfmt +/usr/bin/rebalanceexecutor +/usr/bin/rebalanceplanner +/usr/bin/replicachecker +/usr/bin/requestparser +/usr/bin/sortedhash +/usr/bin/sslfiltertest +/usr/bin/stlset + +/usr/include + +/usr/lib/qfs/libJerasure.a +/usr/lib/qfs/libgf_complete.a +/usr/lib/qfs/libqfs_client.a +/usr/lib/qfs/libqfs_common.a +/usr/lib/qfs/libqfs_emulator.a +/usr/lib/qfs/libqfs_io.a +/usr/lib/qfs/libqfs_meta.a +/usr/lib/qfs/libqfs_qcdio.a +/usr/lib/qfs/libqfs_qcrs.a +/usr/lib/qfs/libqfs_tools.a +/usr/lib/qfs/libqfsc.a +/usr/lib/qfs/libqfskrb.a diff --git a/bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides new file mode 100644 index 0000000000..b1ab3444bb --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides @@ -0,0 +1,2 @@ +qfs-dev binary: new-package-should-close-itp-bug +qfs-dev binary: binary-without-manpage diff --git a/bigtop-packages/src/deb/qfs/qfs-fuse.install b/bigtop-packages/src/deb/qfs/qfs-fuse.install new file mode 100644 index 0000000000..35b2e03574 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-fuse.install @@ -0,0 +1 @@ +/usr/bin/qfs_fuse diff --git a/bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides new file mode 100644 index 0000000000..3daf409abd --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides @@ -0,0 +1,2 @@ +qfs-fuse binary: new-package-should-close-itp-bug +qfs-fuse binary: binary-without-manpage diff --git a/bigtop-packages/src/deb/qfs/qfs-hadoop.install b/bigtop-packages/src/deb/qfs/qfs-hadoop.install new file mode 100644 index 0000000000..c8f20de498 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-hadoop.install @@ -0,0 +1 @@ +/usr/lib/hadoop diff --git a/bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides new file mode 100644 index 0000000000..3e40193882 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides @@ -0,0 +1,2 @@ +qfs-hadoop binary: new-package-should-close-itp-bug +qfs-hadoop binary: jar-not-in-usr-share diff --git a/bigtop-packages/src/deb/qfs/qfs-java.install b/bigtop-packages/src/deb/qfs/qfs-java.install new file mode 100644 index 0000000000..0fb7754a84 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-java.install @@ -0,0 +1 @@ +/usr/share/qfs/java diff --git a/bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides new file mode 100644 index 0000000000..64e9d8a491 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides @@ -0,0 +1 @@ +qfs-java binary: new-package-should-close-itp-bug diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.dirs b/bigtop-packages/src/deb/qfs/qfs-metaserver.dirs new file mode 100644 index 0000000000..725c19aa0c --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.dirs @@ -0,0 +1,2 @@ +/var/log/qfs +/var/lib/qfs diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.install b/bigtop-packages/src/deb/qfs/qfs-metaserver.install new file mode 100644 index 0000000000..c1a57ee700 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.install @@ -0,0 +1,4 @@ +/usr/bin/metaserver +/usr/bin/filelister +/usr/bin/logcompactor +/etc/qfs/logrotate.d/qfs-metaserver diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides new file mode 100644 index 0000000000..a8afec7c04 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides @@ -0,0 +1,2 @@ +qfs-metaserver binary: new-package-should-close-itp-bug +qfs-metaserver binary: binary-without-manpage diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.postinst b/bigtop-packages/src/deb/qfs/qfs-metaserver.postinst new file mode 100644 index 0000000000..f10e01dac0 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.postinst @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + configure) + update-rc.d qfs-metaserver defaults >/dev/null || exit 1 + chown qfs:qfs -R /var/log/qfs + chown qfs:qfs -R /var/run/qfs + chown qfs:qfs -R /var/lib/qfs + ;; + abort-upgrade|abort-remove|abort-deconfigure) + ;; + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.postrm b/bigtop-packages/src/deb/qfs/qfs-metaserver.postrm new file mode 100644 index 0000000000..63ef9c550a --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.postrm @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + purge) + update-rc.d -f qfs-metaserver remove > /dev/null || exit 1 + ;; + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.preinst b/bigtop-packages/src/deb/qfs/qfs-metaserver.preinst new file mode 100644 index 0000000000..7416ecd4a4 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.preinst @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + install|upgrade) + getent group qfs >/dev/null || groupadd -r qfs + getent passwd qfs > /dev/null || useradd -c "qfs" -s /bin/bash -g qfs \ + -d /var/lib/qfs qfs 2> /dev/null || : + ;; + abort-upgrade) + ;; + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# diff --git a/bigtop-packages/src/deb/qfs/qfs-python.install b/bigtop-packages/src/deb/qfs/qfs-python.install new file mode 100644 index 0000000000..5ad7ef7e85 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-python.install @@ -0,0 +1 @@ +/usr/lib/python2.7 diff --git a/bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides new file mode 100644 index 0000000000..ea76f7cb2e --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides @@ -0,0 +1 @@ +qfs-python binary: new-package-should-close-itp-bug diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.dirs b/bigtop-packages/src/deb/qfs/qfs-webui.dirs new file mode 100644 index 0000000000..725c19aa0c --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-webui.dirs @@ -0,0 +1,2 @@ +/var/log/qfs +/var/lib/qfs diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.install b/bigtop-packages/src/deb/qfs/qfs-webui.install new file mode 100644 index 0000000000..767f4fe6b6 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-webui.install @@ -0,0 +1,2 @@ +/usr/share/qfs/webui +/etc/qfs/logrotate.d/qfs-webui diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides new file mode 100644 index 0000000000..d735945846 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides @@ -0,0 +1 @@ +qfs-webui binary: new-package-should-close-itp-bug diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.postinst b/bigtop-packages/src/deb/qfs/qfs-webui.postinst new file mode 100644 index 0000000000..0b0cb5c88f --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-webui.postinst @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + configure) + update-rc.d qfs-webui defaults >/dev/null || exit 1 + chown qfs:qfs -R /var/log/qfs + chown qfs:qfs -R /var/run/qfs + chown qfs:qfs -R /var/lib/qfs + ;; + abort-upgrade|abort-remove|abort-deconfigure) + ;; + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.postrm b/bigtop-packages/src/deb/qfs/qfs-webui.postrm new file mode 100644 index 0000000000..10975ff238 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-webui.postrm @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + purge) + update-rc.d -f qfs-webui remove > /dev/null || exit 1 + ;; + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.preinst b/bigtop-packages/src/deb/qfs/qfs-webui.preinst new file mode 100644 index 0000000000..7416ecd4a4 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs-webui.preinst @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + install|upgrade) + getent group qfs >/dev/null || groupadd -r qfs + getent passwd qfs > /dev/null || useradd -c "qfs" -s /bin/bash -g qfs \ + -d /var/lib/qfs qfs 2> /dev/null || : + ;; + abort-upgrade) + ;; + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# diff --git a/bigtop-packages/src/deb/qfs/qfs.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs.lintian-overrides new file mode 100644 index 0000000000..50904884d6 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/qfs.lintian-overrides @@ -0,0 +1,2 @@ +qfs binary: new-package-should-close-itp-bug +qfs binary: empty-binary-package diff --git a/bigtop-packages/src/deb/qfs/rules b/bigtop-packages/src/deb/qfs/rules new file mode 100755 index 0000000000..56743ea6bb --- /dev/null +++ b/bigtop-packages/src/deb/qfs/rules @@ -0,0 +1,81 @@ +#!/usr/bin/make -f + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +# This has to be exported to make some magic below work. +export DH_OPTIONS + +# Build configuration -- override with environment variables +SHELL := /bin/bash +HADOOP_VERSION ?= 1.0.4 +HADOOP_HOME ?= /usr/lib/hadoop +PYTHON_PATH ?= $(shell command -v python) +PYTHON3_PATH ?= $(shell command -v python3) + +BIN_DIR ?= /usr/bin +LIB_DIR ?= /usr/lib/qfs +ETC_DIR ?= /etc/qfs +INCLUDE_DIR ?= /usr/include +DATA_DIR ?= /usr/share/qfs +VAR_DIR ?= /var + +SERVICES ?= metaserver chunkserver webui + +%: + dh $@ + +override_dh_auto_build: + bash debian/do-component-build \ + --qfs-version=$(QFS_BASE_VERSION) \ + --hadoop-version=$(HADOOP_VERSION) \ + --python=$(PYTHON_PATH) \ + --python3=$(PYTHON3_PATH) + +override_dh_auto_install: + bash -x debian/install_qfs.sh \ + --prefix=debian/tmp \ + --qfs-version=$(QFS_BASE_VERSION) \ + --hadoop-version=$(HADOOP_VERSION) \ + --python=$(PYTHON_PATH) \ + --python3=$(PYTHON3_PATH) \ + --python-extra='--install-layout=deb' \ + --bin-dir=$(BIN_DIR) \ + --lib-dir=$(LIB_DIR) \ + --etc-dir=$(ETC_DIR) \ + --include-dir=$(INCLUDE_DIR) \ + --data-dir=$(DATA_DIR) \ + --var-dir=$(VAR_DIR) + + for service in $(SERVICES); do \ + initd_dir=debian/qfs-$$service/etc/init.d ; \ + mkdir -p $$initd_dir ; \ + bash debian/init.d.tmpl debian/qfs-$$service.svc deb $$initd_dir/qfs-$$service ; \ + done + +override_dh_shlibdeps: + dh_shlibdeps -l$(shell pwd)/build/release/lib + +override_dh_fixperms: + dh_fixperms + # executable-not-elf-or-script + # upstream qfs has executable gifs -- later version fixes but hasn't been + # released yet + chmod -c a-x debian/qfs-webui/usr/share/qfs/webui/files/sorttable/*.gif diff --git a/bigtop-packages/src/deb/qfs/source/format b/bigtop-packages/src/deb/qfs/source/format new file mode 100644 index 0000000000..163aaf8d82 --- /dev/null +++ b/bigtop-packages/src/deb/qfs/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec b/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec new file mode 100644 index 0000000000..599c4ed67c --- /dev/null +++ b/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec @@ -0,0 +1,360 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define qfs_name qfs + +%define bin_dir /usr/bin +%define lib_dir /usr/lib/qfs +%define etc_dir %{_sysconfdir}/qfs +%define include_dir /usr/include +%define data_dir /usr/share/qfs +%define var_dir /var +%define qfs_services chunkserver metaserver webui + +%global HADOOP_VERSION %{hadoop_version} + +%if %{?!HADOOP_VERSION:1}0 +%global HADOOP_VERSION 1.0.4 +%endif + +%if %{?!HADOOP_HOME:1}0 +%global HADOOP_HOME /usr/lib/hadoop +%endif + +# Required for init.d scripts +%global initd_dir %{_sysconfdir}/init.d +%if %{?suse_version:1}0 +Requires: insserv +%global initd_dir %{_sysconfdir}/rc.d +%else +Requires: /lib/lsb/init-functions +%global initd_dir %{_sysconfdir}/rc.d/init.d +%endif + +Name: qfs +Summary: Quantcast Filesystem (QFS) Meta Package +Version: %{qfs_version} +Release: %{qfs_release} +URL: https://quantcast.github.io/qfs +Group: Development/Libraries +License: ASL 2.0 + +BuildRequires: boost-devel >= 1.3.4 +BuildRequires: cmake >= 2.4.7 +BuildRequires: fuse-devel +BuildRequires: gcc-c++ +BuildRequires: krb5-devel +BuildRequires: libuuid-devel +BuildRequires: net-tools +BuildRequires: openssl-devel +BuildRequires: xfsprogs-devel + +Requires: boost >= 1.3.4 +Requires: bigtop-utils +Requires: openssl +Requires: qfs-chunkserver +Requires: qfs-devel +Requires: qfs-fuse +Requires: qfs-hadoop +Requires: qfs-java +Requires: qfs-metaserver +Requires: qfs-python +Requires: qfs-webui + +Source0: %{qfs_name}-%{qfs_base_version}.tar.gz + +%description +Quantcast File System (QFS) is a high-performance, fault-tolerant, distributed +file system developed to support MapReduce processing, or other applications +reading and writing large files sequentially. + +%package client +Group: Applications/System +Summary: Client binaries and libraries used to link against qfs + +%description client +This package provides base client binaries and libraries used to interact with a +running qfs cluster and link against qfs APIs to build your own software that +interacts with a running qfs cluster. + +%package chunkserver +Group: System Environment/Daemons +Summary: Executables required to run the Quantcast File System chunkserver + +%description chunkserver +The QFS chunkserver service hosts the binary contents of the QFS distributed +filesystem. A metaserver coordinates many data nodes running this service, +replicating data amongst chunkservers in a redundant fashion. + + +%package devel +Group: Development/Libraries +Summary: Files needed for building Quantcast File System-based applications + +%description devel +The QFS devel package contains the headers, static libraries, and developer +tool binaries required to develop applications which build against QFS. + + +%package fuse +Group: Applications/System +Summary: Support for mounting the Quantcast File System under FUSE + +%description fuse +This package contains the qfs_fuse executable which is required when mounting +QFS distributed filesystems under FUSE. + + +%package hadoop +Group: Development/Libraries +Summary: Quantcast File System plugin JAR for Hadoop + +%description hadoop +This package contains a plugin JAR to enable QFS to serve as a drop-in +replacement for HDFS under Hadoop. + + +%package java +Group: Development/Libraries +Summary: Java libraries for accessing the Quantcast File System + +%description java +This package contains a JAR which enables Java applications to access QFS via +its JNI interface. + + +%package metaserver +Group: System Environment/Daemons +Summary: Executables required to run the Quantcast File System metaserver + +%description metaserver +This package contains the executables required to run the Quantcast File System +metaserver service, which tracks the location of data chunks distributed across +QFS chunkservers. + + +%package python +Group: Development/Libraries +Summary: Python libraries for accessing the Quantcast File System + +%description python +This package contains the libraries required to access the Quantcast File +System libraries via Python. + + +%package webui +Group: System Environment/Daemons +Requires: python +Summary: Quantcast File System metaserver/chunkserver web frontend + +%description webui +This package contains several Python scripts which provide a simple Hadoop-like +Web UI for viewing Quantcast File Server chunkserver and metaserver status. + +%prep +%setup -n %{qfs_name}-%{qfs_base_version} + +echo $RPM_SOURCE_DIR +echo $RPM_BUILD_ROOT +exit + +%build +%if 0%{?with_python3} +PTYHON3_PATH=%{__python3} +%endif # with_python3 + +bash $RPM_SOURCE_DIR/do-component-build \ + --hadoop-version=%{HADOOP_VERSION} \ + --qfs-version=%{qfs_version} \ + --python=%{__python} \ + --python3=$PYTHON3_PATH + +%install +rm -rf $RPM_BUILD_ROOT +mkdir -p $RPM_BUILD_ROOT + +sh $RPM_SOURCE_DIR/install_qfs.sh \ + --prefix=$RPM_BUILD_ROOT \ + --hadoop-version=%{HADOOP_VERSION} \ + --qfs-version=%{qfs_version} \ + --python=%{__python} \ + --python3=$PYTHON3_PATH \ + --bin-dir=%{bin_dir} \ + --lib-dir=%{lib_dir} \ + --etc-dir=%{etc_dir} \ + --include-dir=%{include_dir} \ + --data-dir=%{data_dir} \ + --var-dir=%{var_dir} + +for service in %{qfs_services} +do + init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{qfs_name}-${service} + bash $RPM_SOURCE_DIR/init.d.tmpl $RPM_SOURCE_DIR/%{qfs_name}-${service}.svc rpm $init_file +done + +# NOTE(fsareshwala): %pre sections copy pasted :( +%pre chunkserver +getent group qfs >/dev/null || groupadd -r qfs +getent passwd qfs >/dev/null || useradd -c "qfs" -s /sbin/nologin -g qfs -r \ + -d %{var_dir}/lib/qfs qfs 2> /dev/null || : + +%pre metaserver +getent group qfs >/dev/null || groupadd -r qfs +getent passwd qfs >/dev/null || useradd -c "qfs" -s /sbin/nologin -g qfs -r \ + -d %{var_dir}/lib/qfs qfs 2> /dev/null || : + +%pre webui +getent group qfs >/dev/null || groupadd -r qfs +getent passwd qfs >/dev/null || useradd -c "qfs" -s /sbin/nologin -g qfs -r \ + -d %{var_dir}/lib/qfs qfs 2> /dev/null || : + +%post chunkserver +/sbin/chkconfig --add qfs-chunkserver + +%preun chunkserver +if [ $1 -eq 0 ]; then + /sbin/service qfs-chunkserver stop &>/dev/null || : + /sbin/chkconfig --del qfs-chunkserver +fi + +%post metaserver +/sbin/chkconfig --add qfs-metaserver + +%preun metaserver +if [ $1 -eq 0 ]; then + /sbin/service qfs-metaserver stop &>/dev/null || : + /sbin/chkconfig --del qfs-metaserver +fi + +%post webui +/sbin/chkconfig --add qfs-webui + +%preun webui +if [ $1 -eq 0 ]; then + /sbin/service qfs-webui stop &>/dev/null || : + /sbin/chkconfig --del qfs-webui +fi + +%clean +rm -rf $RPM_BUILD_ROOT + +%files client +%defattr(-,root,root,755) +%{bin_dir}/qfs +%{bin_dir}/qfsadmin +%{bin_dir}/qfscat +%{bin_dir}/qfsdataverify +%{bin_dir}/qfsfileenum +%{bin_dir}/qfsfsck +%{bin_dir}/qfshibernate +%{bin_dir}/qfsping +%{bin_dir}/qfsput +%{bin_dir}/qfsshell +%{bin_dir}/qfsstats +%{bin_dir}/qfstoggleworm +%{bin_dir}/qfssample +%{bin_dir}/qfs_backup +%{bin_dir}/qfs_checkpoint_prune +%{bin_dir}/qfs_log_prune +%{bin_dir}/cpfromqfs +%{bin_dir}/cptoqfs + +%{lib_dir}/libgf_complete.so +%{lib_dir}/libgf_complete.so.1 +%{lib_dir}/libgf_complete.so.1.0.0 +%{lib_dir}/libJerasure.so +%{lib_dir}/libJerasure.so.2 +%{lib_dir}/libJerasure.so.2.0.0 +%{lib_dir}/libqfs_access.so +%{lib_dir}/libqfs_client.so +%{lib_dir}/libqfs_common.so +%{lib_dir}/libqfsc.so +%{lib_dir}/libqfs_io.so +%{lib_dir}/libqfskrb.so +%{lib_dir}/libqfs_qcdio.so +%{lib_dir}/libqfs_qcrs.so + +%files metaserver +%defattr(-,root,root,-) +%{bin_dir}/metaserver +%{bin_dir}/filelister +%{bin_dir}/logcompactor +%attr(0755,qfs,qfs) %{var_dir}/log/qfs +%attr(0755,qfs,qfs) %{var_dir}/lib/qfs +%{initd_dir}/qfs-metaserver +%config(noreplace) %{etc_dir}/logrotate.d/qfs-metaserver + +%files chunkserver +%defattr(-,root,root,-) +%{bin_dir}/chunkserver +%{bin_dir}/chunkscrubber +%attr(0755,qfs,qfs) %{var_dir}/log/qfs +%attr(0755,qfs,qfs) %{var_dir}/lib/qfs +%{initd_dir}/qfs-chunkserver +%config(noreplace) %{etc_dir}/logrotate.d/qfs-chunkserver + +%files devel +%defattr(-,root,root,-) +%{bin_dir}/checksum +%{bin_dir}/dirtree_creator +%{bin_dir}/dtokentest +%{bin_dir}/qfslogger +%{bin_dir}/rand-sfmt +%{bin_dir}/rebalanceexecutor +%{bin_dir}/rebalanceplanner +%{bin_dir}/replicachecker +%{bin_dir}/requestparser +%{bin_dir}/sortedhash +%{bin_dir}/sslfiltertest +%{bin_dir}/stlset + +%{include_dir}/** + +%{lib_dir}/libJerasure.a +%{lib_dir}/libgf_complete.a +%{lib_dir}/libqfs_client.a +%{lib_dir}/libqfs_common.a +%{lib_dir}/libqfs_emulator.a +%{lib_dir}/libqfs_io.a +%{lib_dir}/libqfs_meta.a +%{lib_dir}/libqfs_qcdio.a +%{lib_dir}/libqfs_qcrs.a +%{lib_dir}/libqfs_tools.a +%{lib_dir}/libqfsc.a +%{lib_dir}/libqfskrb.a + +%files fuse +%defattr(-,root,root,-) +%{bin_dir}/qfs_fuse + +%files hadoop +%defattr(-,root,root,-) +%{HADOOP_HOME}/lib/hadoop-%{HADOOP_VERSION}-qfs-%{qfs_version}.jar + +%files java +%defattr(-,root,root,-) +%{data_dir}/java/qfs-access-%{qfs_version}.jar + +%files python +%defattr(-,root,root,-) +%{python_sitearch}/** + +%files webui +%defattr(-,root,root,-) +%{data_dir}/webui +%attr(0755,qfs,qfs) %{var_dir}/log/qfs +%attr(0755,qfs,qfs) %{var_dir}/lib/qfs +%{initd_dir}/qfs-webui +%config(noreplace) %{etc_dir}/logrotate.d/qfs-webui diff --git a/bigtop-packages/src/templates/init.d.tmpl b/bigtop-packages/src/templates/init.d.tmpl index 8391ad5ad0..7971758b26 100755 --- a/bigtop-packages/src/templates/init.d.tmpl +++ b/bigtop-packages/src/templates/init.d.tmpl @@ -131,7 +131,7 @@ generate_extra_commands() { cat <<'__EOT__' *) - echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart}" + echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart|force-reload}" exit 1 __EOT__ @@ -210,7 +210,7 @@ ERROR_PROGRAM_NOT_CONFIGURED=6 RETVAL=0 SLEEP_TIME=5 -PROC_NAME="java" +PROC_NAME="${PROC_NAME:-java}" DAEMON="$DAEMON" DESC="$DESC" @@ -218,6 +218,8 @@ EXEC_PATH="$EXEC_PATH" SVC_USER="$SVC_USER" DAEMON_FLAGS="$DAEMON_FLAGS" CONF_DIR="$CONF_DIR" +CONF_FILE="$CONF_FILE" +LOGFILE="$LOGFILE" PIDFILE="$PIDFILE" LOCKDIR="/var/lock/subsys" LOCKFILE="\$LOCKDIR/$DAEMON" @@ -291,7 +293,7 @@ service() { checkstatus RETVAL=$? ;; - restart) + restart|force-reload) check_for_root restart ;; diff --git a/bigtop.bom b/bigtop.bom index 363b3fcc16..242cd7f35a 100644 --- a/bigtop.bom +++ b/bigtop.bom @@ -422,5 +422,14 @@ bigtop { site = "${apache.APACHE_MIRROR}/${download_path}" archive = "${apache.APACHE_ARCHIVE}/${download_path}" } } + 'qfs' { + name = 'qfs' + relNotes = 'Quantcast Filesystem' + version { base = '1.1.4'; pkg = base; release = 1 } + tarball { destination = "$name-${version.base}.tar.gz" + source = "${version.base}.tar.gz" } + url { site = "https://github.com/quantcast/qfs/archive" + archive = site } + } } } diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp index f3785301d8..26959e2c0d 100644 --- a/bigtop_toolchain/manifests/packages.pp +++ b/bigtop_toolchain/manifests/packages.pp @@ -22,16 +22,16 @@ } else { $mysql_devel="mariadb-devel" } - $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel" ] + $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ] } - /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel" ] + /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ] # fix package dependencies: BIGTOP-2120 and BIGTOP-2152 exec { '/usr/bin/zypper remove -y krb5-mini': } -> exec {'/usr/bin/zypper install -y libopenssl-devel': } -> Package <| |> } Amazon: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "snappy-devel" ] } - /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev" ] + /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev", "libboost-regex-dev", "xfslibs-dev" ] exec { "apt-update": command => "/usr/bin/apt-get update" diff --git a/packages.gradle b/packages.gradle index 44160ef587..d36e372c4a 100644 --- a/packages.gradle +++ b/packages.gradle @@ -288,6 +288,7 @@ def genTasks = { target -> def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir def final BASE_VERSION = config.bigtop.components[target].version.base def final SRCDEB = "${PKG_NAME}_$PKG_VERSION-${BIGTOP_BUILD_STAMP}.dsc" + def final HADOOP_VERSION = config.bigtop.components["hadoop"].version.pkg exec { workingDir PKG_OUTPUT_DIR @@ -300,6 +301,7 @@ def genTasks = { target -> --preserve-envvar MAVEN3_HOME \ --preserve-envvar MAVEN_OPTS \ --preserve-envvar JAVA_HOME \ +--set-envvar=HADOOP_VERSION=$HADOOP_VERSION \ --set-envvar=${toOldStyleName(target)}_BASE_VERSION=$BASE_VERSION \ --set-envvar=${toOldStyleName(target)}_VERSION=$PKG_VERSION \ --set-envvar=${toOldStyleName(target)}_RELEASE=$BIGTOP_BUILD_STAMP \ @@ -412,12 +414,14 @@ def genTasks = { target -> def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir def final PKG_VERSION = config.bigtop.components[target].version.pkg def final BASE_VERSION = config.bigtop.components[target].version.base + def final HADOOP_VERSION = config.bigtop.components["hadoop"].version.pkg def RELEASE_DIST = "rpmbuild --eval '%{?dist}' 2>/dev/null".execute().text.trim().replaceAll("'",'') def SRCRPM="$PKG_OUTPUT_DIR/$PKG_NAME-${PKG_VERSION}-$BIGTOP_BUILD_STAMP${RELEASE_DIST}.src.rpm" def command = [ '--define', "_topdir $PKG_BUILD_DIR/rpm/", '--define', "${NAME}_base_version $BASE_VERSION", + '--define', "hadoop_version ${HADOOP_VERSION}", '--define', "${NAME}_version ${PKG_VERSION}", '--define', "${NAME}_release ${BIGTOP_BUILD_STAMP}%{?dist}", '--rebuild', SRCRPM,