Find file
Fetching contributors…
Cannot retrieve contributors at this time
executable file 501 lines (452 sloc) 17.7 KB
#!/bin/bash
#
# An apachectl-style script to control a cron scheduled puppet instance
#
# Return codes:
# + 0, everything's fine
# + 1, something went wrong, look for WARNINGs or ERRORs
#
# Based on code by Marco Bonetti <marco.bonetti@slackware.it>
#
# 07/31/2012 bhourigan Added some additional status info to 'puppetctl status'
# 09/21/2012 bhourigan Added a feature to quickly disable puppet for 1 hour when disable is invoked with no arguments
# 09/24/2012 bhourigan Fixed a bug where a sleeping puppet process could run after running 'puppetctl disable'
# 11/07/2012 bhourigan Added a check to see if disabled.at is 0 bytes (disk full) and alert user
# 11/13/2012 bhourigan Added a check to see if last_run_summary.yaml is incomplete, throwing an appropriate error message
# 12/13/2012 bhourigan Fixed a bug where it would erroneously detect invalid characters from ps when multiple puppet instances were running
# 01/10/2013 bhourigan Added posix time spec support, improved command line documentation
# 01/15/2013 bhourigan Fixed two bugs parsing command line parameters, clarified an error message
# 02/26/2013 bhourigan Added logging to help diagnose difficult to troubleshoot bug reports
# 02/27/2013 bhourigan Added a cosmetic enhancement, cleaned up logging functions.
# 03/06/2013 bhourigan Attempting to address bug 830343 again
# 03/12/2013 rsoderberg Added DISABLE_EXPIRATION_TIME, use it in help text as well.
# 03/19/2013 bhourigan Colorized puppet 3.x notices, changed cron detection method from stdin tty to environment variable (thanks :atoll)
# 06/17/2013 rsoderberg Fix new option '-d <datespec>', which takes a time, a date, an ISO time, or +_[mhd].
# 06/24/2013 bhourigan Redirected 'puppet agent --enable' output to /dev/null as recent Puppet changes caused confusion for the user
# 08/19/2013 bhourigan Updated 'puppetctl run' documentation to reflect that options are silently passed to 'puppet agent'
# 09/04/2013 bhourigan Second try at fixing ps output warning bug. There are two places where ps output is sanitized and falsely triggering a warning
# 09/24/2013 bhourigan The disable message is no longer recorded in /etc/motd, instead this has been relocated to a profile.d script. Cleaned up some code
# 01/21/2014 rsoderberg Fix date math for days and include hours [#962051]
# 06/10/2014 bhourigan Puppetctl status now reports last version applied
# 01/12/2015 bhourigan Changing path of puppetctl disable toggle from /var/run/puppetctl.status to /var/lib/puppetctl.status, rebooting RHEL hosts
# had the unintended consequence of re-enabling puppet.
DISABLE_EXPIRATION_TIME='now + 1 hour'
PUPPET_AGENT_OPTIONS='--verbose --onetime --no-daemonize'
log(){
if [ -z "$SUDO_USER" ]; then
logger -t "puppetctl" "$*"
else
logger -t "puppetctl[$SUDO_USER]" "$*"
fi
}
# log_print "message" $color
log_print(){
log "$1"
if [ ! -z "$2" ] && [ -t 0 ]; then
echo -e "\033[${2}m${1}\033[0m"
else
if [ -t 0 ]; then
echo "$1"
else
echo "puppetctl: $1"
fi
fi
}
# error_print "message" $color
error_print(){
log_print "$1" "$2"
exit 1
}
enable(){
rm -f /var/lib/puppetctl.status
if [ -f /var/lib/puppet/state/disabled.at ]; then
if [ -f /var/lib/puppet/state/disabled.at.job ]; then
atrm $(cat /var/lib/puppet/state/disabled.at.job)
rm -f /var/lib/puppet/state/disabled.at.job
fi
rm -f /var/lib/puppet/state/disabled.at
log_print "Puppet has been enabled"
else
log_print "Puppet is already enabled"
fi
}
disable(){
full_args="puppetctl disable"
until [ -z "$1" ]; do
case "$1" in
-m|--message)
if [ -z "$2" ]; then
error_print "missing argument to $1"
fi
full_args="$full_args $1 '${2//\'/\\\'}'"
message="$2"
shift 2
;;
-t|--time)
if [ -z "$2" ]; then
error_print "missing argument to $1"
fi
full_args="$full_args $1 '${2//\'/\\\'}'"
time="$2"
shift 2
;;
-d|--date)
if [ -z "$2" ]; then
error_print "missing argument to $1"
fi
full_args="$full_args $1 '${2//\'/\\\'}'"
time="$2"
parsedate="yes"
shift 2
;;
-T|--posixtime)
if [ -z "$2" ]; then
error_print "missing argument to $1"
fi
full_args="$full_args $1 '${2//\'/\\\'}'"
time="$2"
posix="-t"
shift 2
;;
-f|--force)
full_args="$full_args $1"
force=1
shift 1
;;
*)
error_print "Invalid option: $1"
;;
esac
done
if [[ -n "$parsedate" ]]; then
# at(1) timespecs are terrible. So we force sensible timespecs.
keep_parsing='true'
if $keep_parsing; then
# hh:mm is the easiest. The colon is mandatory.
# H H M M
echo "$time" | grep -q '^[0-2][0-9]:[0-5][0-9]$'
if [[ "${PIPESTATUS[1]}" == "0" ]]; then
# No modification required! at(1) handles this okay.
keep_parsing='false'
fi
fi
if $keep_parsing; then
# YYYYMMDDhhmm is the easiest. Limited to 21st century.
# YY Y Y M M D D H H M M
echo "$time" | grep -q '^20[0-9][0-9][0-1][0-9][0-3][0-9][0-2][0-9][0-5][0-9]$'
if [[ "${PIPESTATUS[1]}" == "0" ]]; then
# No modification required! It's a valid POSIX timestamp.
posix='-t'
keep_parsing='false'
fi
fi
if $keep_parsing; then
# YYYYMMDD is the next easiest. We have to append hhmm 0000 to it.
# YY Y Y M M D D
echo "$time" | grep -q '^20[0-9][0-9][0-1][0-9][0-3][0-9]$'
if [[ "${PIPESTATUS[1]}" == "0" ]]; then
# Modify it to be a valid POSIX timestamp, to prevent MMDDhhmm parsing.
time="${time}0000"
posix='-t'
keep_parsing='false'
fi
fi
if $keep_parsing; then
# +__[m|h|d] is easy, we just need to convert to at timespec.
echo "$time" | grep -q '^\+[0-9]*[mhd]$'
if [[ "${PIPESTATUS[1]}" == "0" ]]; then
# Let's make this an at(1) timespec.
duration="$(echo "$time" | cut -d'+' -f2 | rev | cut -c2- | rev)"
if [[ "$duration" -le 0 ]]; then
echo "Invalid duration argument to -d, aborting."
exit 1
fi
unit="$(echo "$time" | rev | cut -c1 | rev)"
case "$unit" in
m)
unit="minutes"
;;
h)
unit="hours"
;;
d)
unit="days"
;;
*)
unit=
esac
if [[ -z "$unit" ]]; then
echo "Invalid unit argument to -d, aborting."
exit 1
fi
time="now + ${duration} ${unit}"
keep_parsing='false'
fi
fi
if $keep_parsing; then
# Nothing above parsed the date successfully.
echo "Invalid argument to -d, aborting."
exit 1
fi
fi
pid=$(ps auwwx | grep -i 'puppet agent\|puppetd ' | grep -v 'grep\|/bin/sh' | awk '{print $2}' | head -n 1)
if [ ${force:-0} -gt 0 ]; then
security=$(echo $pid | tr -d '[:digit:][:space:]')
if [ ! -z "$security" ]; then
error_print "WARNING: Extra characters detected in pid field from ps: \"$security\" (pid: $pid)"
fi
if [ ! -z "$pid" ]; then
echo -n "Puppet already running as pid $pid, killing with -TERM"
while kill -0 $pid 2>/dev/null; do
kill -TERM $pid 2>/dev/null
echo -n .
sleep 1
done
echo " killed."
log "Puppet already running as pid $pid, killed with -TERM"
fi
else
if [ ! -z "$pid" ]; then
log_print "notice: Run of Puppet configuration client already in progress; skipping (hint: $full_args -f; to kill the running process)" "0;36"
ps auwwx | grep -i 'puppet agent\|puppetd ' | grep -v 'grep\|/bin/sh'
exit 1
fi
fi
if [ -f /var/lib/puppet/state/disabled.at ]; then
if [ ${force:-0} -eq 0 ]; then
log_print "Puppet is already disabled (hint: $full_args -f; to override)"
echo
error_print "$(cat /var/lib/puppetctl.status)" "1;31"
else
if [ -f /var/lib/puppet/state/disabled.at.job ]; then
atrm $(cat /var/lib/puppet/state/disabled.at.job)
rm -f /var/lib/puppet/state/disabled.at.job
fi
rm -f /var/lib/puppetctl.status
fi
fi
if [ -z "$time" ]; then
time="${DISABLE_EXPIRATION_TIME}"
fi
if [ ! -z "$SUDO_USER" ]; then
invoking_user="$SUDO_USER"
else
invoking_user="$USER"
fi
echo "logger -t \"puppetctl[$invoking_user]\" Puppet has been enabled on schedule" > /var/lib/puppet/state/disabled.at
echo "rm -f /var/lib/puppet/state/disabled.at /var/lib/puppetctl.status" >> /var/lib/puppet/state/disabled.at
if [ ! -s /var/lib/puppet/state/disabled.at ]; then
enable
error_print "Disk might be full. Can't write to /var/lib/puppet/state/disabled.at. Refusing to disable puppet." "1;31"
fi
output=$(at -f /var/lib/puppet/state/disabled.at ${posix:-} "$time" 2>&1)
status=$?
if [ $status -ne 0 ]; then
log_print "ERROR: at returned non-zero exit status $status"
echo
echo "$output"
enable
exit 1
else
job=$(echo "$output" | sed -e 's/job \([[:digit:]]*\) at.*/\1/g')
if [ ! -z "$job" ]; then
echo $job > /var/lib/puppet/state/disabled.at.job
if [ ! -s /var/lib/puppet/state/disabled.at.job ]; then
enable
error_print "Disk might be full. Can't write to /var/lib/puppet/state/disabled.at.job. Refusing to disable puppet." "1;31"
fi
fi
fi
real_time=$(echo "$output" | sed '/^job.*at */!d; s///;q')
if [ -z "$message" ]; then
string="Puppet has been disabled by $invoking_user at $(date "+%Y-%m-%d %H:%M") until $real_time"
else
string="Puppet has been disabled by $invoking_user at $(date "+%Y-%m-%d %H:%M") until $real_time with the following message: $message"
fi
echo "$string" > /var/lib/puppetctl.status
if [ ! -s /var/lib/puppetctl.status ]; then
enable
error_print "Disk might be full. Can't write to /var/lib/puppetctl.status. Refusing to disable puppet." "1;31"
fi
log_print "$string" "1;31"
}
run(){
if [ ! -f /var/lib/puppet/state/disabled.at ]; then
pid=$(ps auwwx | grep -i 'puppet agent' | grep -v 'grep\|/bin/sh' | awk '{print $2}')
security=$(echo $pid | tr -d '[:digit:][:space:]')
if [ ! -z "$security" ]; then
error_print "WARNING: Extra characters detected in pid field from ps: \"$security\" (pid: $pid)"
fi
if [ ! -z "$pid" ]; then
age=$(stat -c '%Y' /proc/$pid)
current=$(date "+%s")
delta=$(expr $current - $age)
if [ ! -z "$delta" ] && [ "$delta" -gt 3600 ]; then
log_print "Killed hung puppet process, had been running for $delta seconds"
kill -9 $pid
fi
fi
puppet agent --enable >/dev/null 2>&1
if [ -t 0 ] || [ -z "$MAILTO" ]; then
puppet agent $PUPPET_AGENT_OPTIONS --no-splay $@
else
puppet agent $PUPPET_AGENT_OPTIONS $@
fi
else
if [ -t 0 ]; then
grep -v '^[ ]*#' /etc/cron.d/puppetcheck | grep -q 'puppet agent\|puppetctl'
if [ $? -ne 0 ]; then
error_print "Puppet has not been disabled with puppetctl, but it seems to be commented out in /etc/cron.d/puppetcheck. Not running puppet."
else
error_print "$(cat /var/lib/puppetctl.status)" "1;31"
fi
fi
fi
}
status(){
if [ ! -f /var/lib/puppet/state/disabled.at ]; then
if [ -f /etc/cron.d/puppetcheck ]; then
grep -v '^[ ]*#' /etc/cron.d/puppetcheck | grep -q 'puppet agent\|puppetctl'
if [ $? -eq 0 ]; then
if [ -x /usr/bin/puppet ]; then
if [ -f /var/lib/puppet/state/last_run_summary.yaml ]; then
age=$(ruby -ryaml -e "output = File.open('/var/lib/puppet/state/last_run_summary.yaml'){ |data| YAML::load(data) }; puts Time.now.to_i - output['time']['last_run'].to_i" 2>/dev/null)
errors=$(ruby -ryaml -e "output = File.open('/var/lib/puppet/state/last_run_summary.yaml'){ |data| YAML::load(data) }; puts output['resources']['failed']" 2>/dev/null)
config=$(ruby -ryaml -e "output = File.open('/var/lib/puppet/state/last_run_summary.yaml'){ |data| YAML::load(data) }; puts output['version']['config']" 2>/dev/null)
if [ -z "$age" ] || [ -z "$errors" ]; then
log_print "Puppet is enabled, but /var/lib/puppet/state/last_run_summary.yaml is incomplete (puppet is broken)"
else
if [[ "$age" -lt 1 ]]; then
# Handle zero or negative $age gracefully.
l_result="${age}s"
else
# Fancy logic to emit "4d 1m" instead of "4d 0h 1m 0s".
declare -a last_ran
l_days="$(($age/86400))"
if [[ "$l_days" -ne 0 ]]; then
last_ran+=("${l_days}d")
fi
l_hours="$(($age%86400/3600))"
if [[ "$l_hours" -ne 0 ]]; then
last_ran+=("${l_hours}h")
fi
l_minutes="$(($age%3600/60))"
if [[ "$l_minutes" -ne 0 ]]; then
last_ran+=("${l_minutes}m")
fi
l_seconds="$(($age%60))"
if [[ "$l_seconds" -ne 0 ]]; then
last_ran+=("${l_seconds}s")
fi
oldifs="$IFS"
IFS=" "
l_result="${last_ran[*]}"
IFS="$oldifs"
fi
log_print "Puppet is enabled, last ran ${l_result} ago with $errors errors, applied version ${config}"
fi
else
log_print "Puppet is enabled"
fi
exit 0
else
error_print "Puppet has not been disabled with puppetctl, but it seems /usr/bin/puppet is not set to be executable"
fi
else
error_print "Puppet has not been disabled with puppetctl, but puppet seems to be commented out in /etc/cron.d/puppetcheck"
fi
else
error_print "Puppet has not been disabled with puppetctl, but /etc/cron.d/puppetcheck seems to be missing"
fi
else
error_print "$(cat /var/lib/puppetctl.status)" "1;31"
fi
}
usage() {
if [ $# -gt 0 ]; then
echo "ERROR: $*"
echo
fi
echo -e "Usage: $0 <command> [<options>]"
echo
echo "Commands:"
echo "---------"
echo " enable Enable puppet"
echo " disable Disable puppet"
echo " run Puppet agent run"
echo " status Enable puppet"
echo
echo "Options:"
echo "--------"
echo " enable"
echo " No options"
echo
echo " disable"
echo " -t"
echo " --time 'at(1) timespec'"
echo " Set the disable expiration time using at(1) specification. Defaults"
echo " to '${DISABLE_EXPIRATION_TIME}'"
echo
echo " -T"
echo " --posixtime [[CC]YY]MMDDhhmm[.SS]"
echo " Set the disable expiration time using POSIX time format instead."
echo
echo " -d"
echo " --date FORMAT"
echo " Set the disable expiration time using one of these formats:"
echo " Duration: +__m, +__h, +__d (minutes, hours, days from now)"
echo " Time: hh:mm, YYYYMMDDhhmm"
echo " Date: YYYYMMDD (at midnight)"
echo
echo " -f"
echo " --force"
echo " Force disable. If puppet is running it will be terminated, it will"
echo " override any existing disable"
echo
echo " -m"
echo " --message"
echo " Set a message that will be displayed in /etc/motd and any time the"
echo " status is queried with puppetctl."
echo
echo " run"
echo " Arguments are passed directly to the puppet agent command. Puppetctl itself has no options."
echo
echo " status"
echo " No options"
echo
echo "Examples:"
echo "--------"
echo "# puppetctl disable -t 'now + 2 days' -m 'Disabled for 2 days'"
echo "# puppetctl disable -f -m 'Disabled for 1 hour, killing puppet if its running'"
echo "# puppetctl disable -T 201510211629 -m 'I will buy you a beer if you recognize the date'"
echo "# puppetctl run --noop"
echo
echo "Want more examples? File a bug!"
exit 1
}
# check root
if [ ! "${EUID}" = "0" ]; then
echo "You need root privileges to run $0"
exit 1
fi
log "Running with: $*"
case "$1" in
'enable')
enable
;;
'disable')
shift 1
disable "$@"
;;
'run')
shift 1
run "$@"
;;
'status')
status
;;
*)
usage
;;
esac
exit 0