Skip to content
Permalink
Browse files
BIGTOP-3484. Bump Alluxio to 2.4.1. (#724)
  • Loading branch information
sekikn committed Feb 12, 2021
1 parent 8c81fcb commit 8db007af74b2f7a44a2d8ed2909f08ca15331a36
Show file tree
Hide file tree
Showing 13 changed files with 184 additions and 49 deletions.
@@ -57,6 +57,13 @@
hasrestart => true,
hasstatus => true,
}
service { "alluxio-job-master":
ensure => running,
require => [ Package["alluxio"], Exec["daemon-reload"], Exec["alluxio formatting"] ],
subscribe => File["/etc/alluxio/conf/alluxio-site.properties"],
hasrestart => true,
hasstatus => true,
}
}

}
@@ -77,5 +84,12 @@
hasrestart => true,
hasstatus => true,
}
service { "alluxio-job-worker":
ensure => running,
require => [ Package["alluxio"], Exec["daemon-reload"], File["/etc/alluxio/conf/alluxio-site.properties"] ],
subscribe => File["/etc/alluxio/conf/alluxio-site.properties"],
hasrestart => true,
hasstatus => true,
}
}
}
@@ -0,0 +1,75 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

TYPE="job-master"
DAEMON="alluxio-${TYPE}"
DESC="Alluxio ${TYPE}"
EXEC_PATH="/usr/lib/alluxio/bin"
WORKING_DIR="/var/lib/alluxio"
DAEMON_FLAGS="job_master"
KEYWORD="alluxio.master.AlluxioJobMaster"
CONF_DIR="/etc/alluxio/conf"
PIDFILE="/var/run/alluxio/${DAEMON}.pid"

generate_start() {

cat <<'__EOT__'
start() {
[ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
log_success_msg "Starting $DESC (${DAEMON}): "

checkstatusofproc
status=$?
if [ "$status" -eq "$STATUS_RUNNING" ]; then
log_success_msg "${DESC} is running"
exit 0
fi

LOG_FILE=/var/log/alluxio/${DAEMON}.out

/bin/bash -c "nohup nice -n 0 \
${EXEC_PATH}/alluxio-start.sh ${DAEMON_FLAGS} \
> $LOG_FILE 2>&1 & "

WAIT_TIMES=0
until ps -A -o pid,command | grep -i "[j]ava" | grep $KEYWORD || [ $WAIT_TIMES -eq 10 ]; do
(( WAIT_TIMES++ ))
sleep 1
done
echo `ps -A -o pid,command | grep -i "[j]ava" | grep $KEYWORD | awk '{print $1}'` > $PIDFILE

checkstatusofproc
RETVAL=$?
[ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE
return $RETVAL
}
__EOT__

}

generate_stop() {

cat <<'__EOT__'
stop() {
log_success_msg "Stopping $DESC (${DAEMON}): "
${EXEC_PATH}/alluxio-stop.sh ${DAEMON_FLAGS} 2>&1 > /dev/null
RETVAL=$?

[ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
return $RETVAL
}
__EOT__

}
@@ -0,0 +1,79 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

TYPE="job-worker"
DAEMON="alluxio-${TYPE}"
DESC="Alluxio ${TYPE}"
EXEC_PATH="/usr/lib/alluxio/bin"
WORKING_DIR="/var/lib/alluxio"
DAEMON_FLAGS="job_worker"
KEYWORD="alluxio.worker.AlluxioJobWorker"
CONF_DIR="/etc/alluxio/conf"
PIDFILE="/var/run/alluxio/${DAEMON}.pid"

generate_start() {

cat <<'__EOT__'
start() {
[ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
log_success_msg "Starting $DESC (${DAEMON}): "

checkstatusofproc
status=$?
if [ "$status" -eq "$STATUS_RUNNING" ]; then
log_success_msg "${DESC} is running"
exit 0
fi

LOG_FILE=/var/log/alluxio/${DAEMON}.out

if [ -f $CONF_DIR/alluxio-env.sh ]; then
. $CONF_DIR/alluxio-env.sh
fi

/bin/bash -c "nohup nice -n 0 \
${EXEC_PATH}/alluxio-start.sh ${DAEMON_FLAGS} \
> $LOG_FILE 2>&1 & "

WAIT_TIMES=0
until ps -A -o pid,command | grep -i "[j]ava" | grep $KEYWORD || [ $WAIT_TIMES -eq 30 ]; do
(( WAIT_TIMES++ ))
sleep 1
done
echo `ps -A -o pid,command | grep -i "[j]ava" | grep $KEYWORD | awk '{print $1}'` > $PIDFILE

checkstatusofproc
RETVAL=$?
[ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE
return $RETVAL
}
__EOT__

}

generate_stop() {

cat <<'__EOT__'
stop() {
log_success_msg "Stopping $DESC (${DAEMON}): "
${EXEC_PATH}/alluxio-stop.sh ${DAEMON_FLAGS} 2>&1 > /dev/null
RETVAL=$?

[ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
return $RETVAL
}
__EOT__

}
@@ -48,7 +48,7 @@ start() {
> $LOG_FILE 2>&1 & "

WAIT_TIMES=0
until ps -A -o pid,command | grep -i "[j]ava" | grep $KEYWORD || [ $WAIT_TIMES -eq 10 ]; do
until ps -A -o pid,command | grep -i "[j]ava" | grep $KEYWORD || [ $WAIT_TIMES -eq 30 ]; do
(( WAIT_TIMES++ ))
sleep 1
done
@@ -18,8 +18,4 @@ set -ex

. `dirname $0`/bigtop.bom

t="hadoop.version"
sed -i "s#<$t>.*</$t>#<$t>${HADOOP_VERSION}</$t>#" pom.xml
sed -i "s|http://repo.spring.io|https://repo.spring.io|g" pom.xml

mvn clean install -DskipTests -Pspark -Pyarn "$@"
mvn clean install -DskipTests -Dhadoop.version=3.2.2 -Phadoop-3 -Pyarn "$@"
@@ -124,14 +124,12 @@ cp -a lib/* $PREFIX/$LIB_DIR/lib
cp -a libexec/* $PREFIX/$LIB_DIR/libexec
cp -a client/* $PREFIX/$LIB_DIR/client
cp -a integration/* $PREFIX/$LIB_DIR/integration
cp integration/checker/target/alluxio-checker-*-jar-with-dependencies.jar $PREFIX/$LIB_DIR/integration/checker
cp integration/fuse/target/alluxio-integration-fuse-*-jar-with-dependencies.jar $PREFIX/$LIB_DIR/integration/fuse
cp integration/yarn/target/alluxio-integration-yarn-*-jar-with-dependencies.jar $PREFIX/$LIB_DIR/integration/yarn
rm -rf $PREFIX/$LIB_DIR/integration/pom.xml $PREFIX/$LIB_DIR/integration/**/pom.xml
rm -rf $PREFIX/$LIB_DIR/integration/target $PREFIX/$LIB_DIR/integration/**/target
rm -rf $PREFIX/$LIB_DIR/integration/**/src
rm -rf $PREFIX/$LIB_DIR/integration/**/README.md
cp -rf core/server/common/src/main/webapp $PREFIX/$LIB_DIR/core/server/common/src/main

# Copy in the configuration files
install -m 0644 conf/log4j.properties $PREFIX/$CONF_DIST_DIR
@@ -145,9 +143,6 @@ cp conf/workers $PREFIX/$CONF_DIST_DIR/workers
# Copy in the /usr/bin/alluxio wrapper
install -d -m 0755 $PREFIX/$BIN_DIR

# Copy in alluxio deploy scripts
cp -rf deploy $PREFIX/$LIB_DIR/share

# Prefix is correct at time of install,
# but we dont want to escape it before that point.
cat > $PREFIX/$BIN_DIR/alluxio <<EOF
@@ -1,10 +1,10 @@
diff --git a/pom.xml b/pom.xml
index a959527aa2..1051f153e2 100644
index 8b6440b7e7..4ecbca4be4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1175,6 +1175,7 @@

@@ -1214,6 +1214,7 @@
<!-- Build and Packaging Exclusions -->
<exclude>webui/node_modules/*</exclude>
<exclude>build/**/*</exclude>
+ <exclude>debian/**/*</exclude>
<exclude>conf/alluxio-env.sh</exclude>

This file was deleted.

This file was deleted.

@@ -41,3 +41,5 @@ override_dh_auto_install:
mkdir -p debian/alluxio/etc/init.d/
bash debian/init.d.tmpl debian/alluxio-master.svc deb debian/alluxio/etc/init.d/alluxio-master
bash debian/init.d.tmpl debian/alluxio-worker.svc deb debian/alluxio/etc/init.d/alluxio-worker
bash debian/init.d.tmpl debian/alluxio-job-master.svc deb debian/alluxio/etc/init.d/alluxio-job-master
bash debian/init.d.tmpl debian/alluxio-job-worker.svc deb debian/alluxio/etc/init.d/alluxio-job-worker
@@ -29,9 +29,11 @@ Source2: install_alluxio.sh
Source3: init.d.tmpl
Source4: alluxio-master.svc
Source5: alluxio-worker.svc
Source6: alluxio-job-master.svc
Source7: alluxio-job-worker.svc
#BIGTOP_PATCH_FILES
%define alluxio_home /usr/lib/%{alluxio_name}
%define alluxio_services master worker
%define alluxio_services master worker job-master job-worker
%define var_lib /var/lib/%{alluxio_name}
%define var_run /var/run/%{alluxio_name}
%define var_log /var/log/%{alluxio_name}
@@ -136,6 +138,8 @@ done
%config(noreplace) %{_sysconfdir}/%{alluxio_name}/conf.dist/core-site.xml
%config(noreplace) %{initd_dir}/%{alluxio_name}-master
%config(noreplace) %{initd_dir}/%{alluxio_name}-worker
%config(noreplace) %{initd_dir}/%{alluxio_name}-job-master
%config(noreplace) %{initd_dir}/%{alluxio_name}-job-worker
%attr(0755,alluxio,alluxio) %{var_lib}
%attr(0755,alluxio,alluxio) %{var_run}
%attr(0755,alluxio,alluxio) %{var_log}
@@ -66,11 +66,12 @@ class TestAlluxioSmoke {
sh.exec("""
set -x
# Clear test environment
$hadoop fs -rm $alluxioTestDir/hadoopLs/datafile
$alluxio fs rm $alluxioTestDir/hadoopLs/datafile
$hadoop fs -rm -r /underFSStorage/$alluxioTestDir
$alluxio fs rm -R $alluxioTestDir
set -e
# Test Alluxio and HDFS interoperability
$alluxio fs mkdir $alluxioTestDir/hadoopLs
$alluxio fs copyFromLocal datafile $alluxioTestDir/hadoopLs/datafile
$alluxio fs persist $alluxioTestDir/hadoopLs/datafile
$hadoop fs -cat /underFSStorage/$alluxioTestDir/hadoopLs/datafile
@@ -352,7 +352,7 @@ bigtop {
name = "alluxio"
pkg = "alluxio"
relNotes = "Alluxio: a memory-centric distributed file system"
version { base = '1.8.2'; pkg = base; release = 1 }
version { base = '2.4.1'; pkg = base; release = 1 }
tarball { destination = "$name-${version.base}.tar.gz"
source = "v${version.base}.tar.gz" }
url { site = "https://github.com/Alluxio/alluxio/archive"

0 comments on commit 8db007a

Please sign in to comment.