From e9a2551b315b4395c5227dad017f2b4340f41108 Mon Sep 17 00:00:00 2001 From: Kunal Khatua Date: Mon, 2 Apr 2018 22:35:53 -0700 Subject: [PATCH 1/2] DRILL-143: Support CGROUPs resource management Introduces the DRILLBIT_CGROUP option in drill-env.sh. The startup script checks if the specified CGroup (ver 2) is available and tries to apply it to the launched Drillbit JVM. This would benefit not just Drill-on-YARN usecases, but any setup that would like CGroups for enforcement of (cpu) resources management. e.g when Drillbit is configured to use `drillcpu` cgroup ``` [root@maprlabs ~]# /opt/mapr/drill/apache-drill-1.14.0-SNAPSHOT/bin/drillbit.sh restart Stopping drillbit .. Starting drillbit, logging to /var/log/drill/drillbit.out WARN: Drillbit's CPU resource usage will be managed under the CGroup : drillcpu (up to 4.00 cores allowed) ``` e.g. Non-existent CGroup `droolcpu` is used ``` [root@kk127 ~]# /opt/mapr/drill/apache-drill-1.14.0-SNAPSHOT/bin/drillbit.sh restart Stopping drillbit .. Starting drillbit, logging to /var/log/drill/drillbit.out ERROR: cgroup droolcpu does not found. Ensure that daemon is running and cgroup exists ``` --- distribution/src/resources/drill-env.sh | 6 ++++ distribution/src/resources/drillbit.sh | 39 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/distribution/src/resources/drill-env.sh b/distribution/src/resources/drill-env.sh index 351d8673681..a1fc1f34af1 100755 --- a/distribution/src/resources/drill-env.sh +++ b/distribution/src/resources/drill-env.sh @@ -86,6 +86,12 @@ #export DRILL_PID_DIR=${DRILL_PID_DIR:-$DRILL_HOME} +# CGroup to which the Drillbit belong when running as a daemon using +# drillbit.sh start. +# Unset $DRILLBIT_CGROUP by default. + +#export DRILLBIT_CGROUP=${DRILLBIT_CGROUP:-"drillcpu"} + # Custom JVM arguments to pass to the both the Drillbit and sqlline. Typically # used to override system properties as shown below. Empty by default. diff --git a/distribution/src/resources/drillbit.sh b/distribution/src/resources/drillbit.sh index 11a124ec55e..ee55e34e81a 100755 --- a/distribution/src/resources/drillbit.sh +++ b/distribution/src/resources/drillbit.sh @@ -127,6 +127,44 @@ check_before_start() fi } +check_after_start(){ + #check if the process is running + if [ -f $pid ]; then + dbitProc=$(ps -ef | grep `cat $pid` | grep Drillbit) + if [ -n "$dbitProc" ]; then + # Check and enforce for CGroup + if [ -n "$DRILLBIT_CGROUP" ]; then + check_and_enforce_cgroup `cat $pid` + fi + fi + fi +} + +check_and_enforce_cgroup(){ + dbitPid=$1; + #if [ $(`ps -o cgroup` | grep -c $DRILLBIT_CGROUP ) -eq 1 ]; then + if [ -f /cgroup/cpu/${DRILLBIT_CGROUP}/cgroup.procs ]; then + echo $dbitPid > /cgroup/cpu/${DRILLBIT_CGROUP}/cgroup.procs + # Verify Enforcement + cgroupStatus=`grep -w $pid /cgroup/cpu/${DRILLBIT_CGROUP}/cgroup.procs` + if [ -z "$cgroupStatus" ]; then + #Ref: https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt + let cpu_quota=`cat /cgroup/cpu/${DRILLBIT_CGROUP}/cpu.cfs_quota_us` + let cpu_period=`cat /cgroup/cpu/${DRILLBIT_CGROUP}/cpu.cfs_period_us` + if [ $cpu_period -gt 0 ] && [ $cpu_quota -gt 0 ]; then + coresAllowed="(up to "`echo $(( 100 * $cpu_quota / $cpu_period )) | sed 's/..$/.&/'`" cores allowed)" + fi + echo "WARN: Drillbit's CPU resource usage will be managed under the CGroup : $DRILLBIT_CGROUP "$coresAllowed + else + echo "ERROR: Failed to add Drillbit to CGroup ( $DRILLBIT_CGROUP ) for resource usage management. Ensure that the cgroup manages CPU" + exit 1 + fi + else + echo "ERROR: cgroup $DRILLBIT_CGROUP does not found. Ensure that daemon is running and cgroup exists" + exit 1 + fi +} + wait_until_done () { p=$1 @@ -154,6 +192,7 @@ start_bit ( ) nohup nice -n $DRILL_NICENESS "$DRILL_HOME/bin/runbit" exec ${args[@]} >> "$logout" 2>&1 & echo $! > $pid sleep 1 + check_after_start } stop_bit ( ) From a5bbed8669e76bc068da0845e15974e342986404 Mon Sep 17 00:00:00 2001 From: Kunal Khatua Date: Wed, 4 Apr 2018 15:39:54 -0700 Subject: [PATCH 2/2] Updates: Changes based on comments 1. Introduced SYS_CGROUP_DIR to account for possible locations of CGroup 2. Removed exit codes for successful Drillbit startups but failed CGroup assignment. 3. General code cleanup --- distribution/src/resources/drill-env.sh | 10 ++++-- distribution/src/resources/drillbit.sh | 44 ++++++++++++------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/distribution/src/resources/drill-env.sh b/distribution/src/resources/drill-env.sh index a1fc1f34af1..4dd52aa1f55 100755 --- a/distribution/src/resources/drill-env.sh +++ b/distribution/src/resources/drill-env.sh @@ -86,10 +86,14 @@ #export DRILL_PID_DIR=${DRILL_PID_DIR:-$DRILL_HOME} -# CGroup to which the Drillbit belong when running as a daemon using -# drillbit.sh start. -# Unset $DRILLBIT_CGROUP by default. +# Default (Standard) CGroup Location: /sys/fs/cgroup +# Specify the cgroup location if it is different from the default +#export SYS_CGROUP_DIR=${SYS_CGROUP_DIR:-"/sys/fs/cgroup"} + +# CGroup to which the Drillbit belongs when running as a daemon using drillbit.sh start . +# Drill will use CGroup for CPU enforcement only. +# Unset $DRILLBIT_CGROUP by default #export DRILLBIT_CGROUP=${DRILLBIT_CGROUP:-"drillcpu"} # Custom JVM arguments to pass to the both the Drillbit and sqlline. Typically diff --git a/distribution/src/resources/drillbit.sh b/distribution/src/resources/drillbit.sh index ee55e34e81a..3634753cdcc 100755 --- a/distribution/src/resources/drillbit.sh +++ b/distribution/src/resources/drillbit.sh @@ -128,40 +128,38 @@ check_before_start() } check_after_start(){ - #check if the process is running - if [ -f $pid ]; then - dbitProc=$(ps -ef | grep `cat $pid` | grep Drillbit) - if [ -n "$dbitProc" ]; then - # Check and enforce for CGroup - if [ -n "$DRILLBIT_CGROUP" ]; then - check_and_enforce_cgroup `cat $pid` - fi - fi + dbitPid=$1; + # Check and enforce for CGroup + if [ -n "$DRILLBIT_CGROUP" ]; then + check_and_enforce_cgroup $dbitPid fi } check_and_enforce_cgroup(){ dbitPid=$1; - #if [ $(`ps -o cgroup` | grep -c $DRILLBIT_CGROUP ) -eq 1 ]; then - if [ -f /cgroup/cpu/${DRILLBIT_CGROUP}/cgroup.procs ]; then - echo $dbitPid > /cgroup/cpu/${DRILLBIT_CGROUP}/cgroup.procs + kill -0 $dbitPid + if [ $? -gt 0 ]; then + echo "ERROR: Failed to add Drillbit to CGroup ( $DRILLBIT_CGROUP ) for 'cpu'. Ensure that the Drillbit ( pid=$dbitPid ) started up." >&2 + exit 1 + fi + SYS_CGROUP_DIR=${SYS_CGROUP_DIR:-"/sys/fs/cgroup"} + if [ -f $SYS_CGROUP_DIR/cpu/$DRILLBIT_CGROUP/cgroup.procs ]; then + echo $dbitPid > $SYS_CGROUP_DIR/cpu/$DRILLBIT_CGROUP/cgroup.procs # Verify Enforcement - cgroupStatus=`grep -w $pid /cgroup/cpu/${DRILLBIT_CGROUP}/cgroup.procs` + cgroupStatus=`grep -w $pid $SYS_CGROUP_DIR/cpu/${DRILLBIT_CGROUP}/cgroup.procs` if [ -z "$cgroupStatus" ]; then #Ref: https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt - let cpu_quota=`cat /cgroup/cpu/${DRILLBIT_CGROUP}/cpu.cfs_quota_us` - let cpu_period=`cat /cgroup/cpu/${DRILLBIT_CGROUP}/cpu.cfs_period_us` + cpu_quota=`cat ${SYS_CGROUP_DIR}/cpu/${DRILLBIT_CGROUP}/cpu.cfs_quota_us` + cpu_period=`cat ${SYS_CGROUP_DIR}/cpu/${DRILLBIT_CGROUP}/cpu.cfs_period_us` if [ $cpu_period -gt 0 ] && [ $cpu_quota -gt 0 ]; then - coresAllowed="(up to "`echo $(( 100 * $cpu_quota / $cpu_period )) | sed 's/..$/.&/'`" cores allowed)" + coresAllowed=`echo $(( 100 * $cpu_quota / $cpu_period )) | sed 's/..$/.&/'` + echo "INFO: CGroup (drillcpu) will limit Drill to $coresAllowed cpu(s)" fi - echo "WARN: Drillbit's CPU resource usage will be managed under the CGroup : $DRILLBIT_CGROUP "$coresAllowed else - echo "ERROR: Failed to add Drillbit to CGroup ( $DRILLBIT_CGROUP ) for resource usage management. Ensure that the cgroup manages CPU" - exit 1 + echo "ERROR: Failed to add Drillbit to CGroup ( $DRILLBIT_CGROUP ) for 'cpu'. Ensure that the cgroup manages 'cpu'" >&2 fi else - echo "ERROR: cgroup $DRILLBIT_CGROUP does not found. Ensure that daemon is running and cgroup exists" - exit 1 + echo "ERROR: CGroup $DRILLBIT_CGROUP not found. Ensure that daemon is running, SYS_CGROUP_DIR is correctly set (currently, $SYS_CGROUP_DIR ), and that the CGroup exists" >&2 fi } @@ -190,9 +188,11 @@ start_bit ( ) echo "`date` Starting $command on `hostname`" >> "$DRILLBIT_LOG_PATH" echo "`ulimit -a`" >> "$DRILLBIT_LOG_PATH" 2>&1 nohup nice -n $DRILL_NICENESS "$DRILL_HOME/bin/runbit" exec ${args[@]} >> "$logout" 2>&1 & + procId=$! + echo $procId > $pid # Yeah, $pid is a file, $procId is the pid... echo $! > $pid sleep 1 - check_after_start + check_after_start $procId } stop_bit ( )