foreman-debug

#!/bin/bash
# vim:sw=2:ts=2:et

export SCLNAME=tfm

usage() {
cat <<USAGE
$0 - configuration and log data collector

USAGE: $0 [options]

Collects configuration and log data for Foreman, Smart Proxies, backend
services and system information while removing security information like
passwords, tokens and keys.

This program can be used on Foreman instances, Smart Proxy instances or
backend services separately.

OPTIONS:
  -d DIR  Directory to place the tarball in (default /var/tmp/foreman-XYZ)
  -g      Skip generic info (CPU, memory, firewall etc.)
  -a      Keep directory and do not generate a compressed tarball
  -s SIZE Maximum log set size in MB (current+rotated files, defaults to 10 MB)
  -j PRG  Filter with provided program when creating a tarball
  -p      Additionally print password patterns being filtered out
  -q      Quiet mode
  -v      Verbose mode
  -h      Shows this message

USAGE
}

# Filter for patterns like password=XYZ, -storepass XYZ or secret: XYZ
FILTER_WORDS=(
password
PASSWORD
default_password
oauth_consumer_key
secret
token
keystorePass
truststorePass
storepass
)
FILTER_WORDS_STR=$(IFS=$'|'; echo "${FILTER_WORDS[*]}")
FILTER="s/($FILTER_WORDS_STR)(\"?\s*[:=]?\s*)\S+/\1\2+FILTERED+/g"

error() {
  echo "$*" >&2
}

qprintf() {
  [ $QUIET -ne 1 ] && printf "$@"
}

printv() {
  [ $QUIET -ne 1 ] && [ $VERBOSE -eq 1 ] && echo "[$SECONDS]" $*
}

clean_stdin() {
  while read -e -t 0.1; do : ; done
}

# add outout of the command and redirect possible errors there
add_cmd() {
  CMD=$1
  OUT=$2
  printv " - $OUT"
  echo -e "COMMAND> $CMD\n" > "$DIR/$OUT"
  eval $CMD >> "$DIR/$OUT" 2>&1
}

# add and filter file of known MIME type, calculate size
add_file() {
  FILE=$1
  SIZE=$(stat -c "%s" $FILE)
  MIME=$(file -bi "$FILE" | cut -d\; -f1)
  case $MIME in
        application/x-gzip)
          OUTFILE="$DIR${FILE/%.gz/}"
          zcat "$FILE" | sed -r "$FILTER" > "$OUTFILE"
          touch -c -r "$FILE" "$OUTFILE"
          ;;
        application/x-bzip2)
          OUTFILE="$DIR${FILE/%.bz2/}"
          bzcat "$FILE" | sed -r "$FILTER" > "$OUTFILE"
          touch -c -r "$FILE" "$OUTFILE"
          ;;
        application/x-xz)
          OUTFILE="$DIR${FILE/%.xz/}"
          xzcat "$FILE" | sed -r "$FILTER" > "$OUTFILE"
          touch -c -r "$FILE" "$OUTFILE"
          ;;
        text/plain | application/xml)
          OUTFILE="$DIR$FILE"
          sed -r "$FILTER" "$FILE" > "$OUTFILE"
          [ $PRINTPASS -eq 1 ] && grep -H "+FILTERED+" "$OUTFILE"
          touch -c -r "$FILE" "$OUTFILE"
          ;;
        *)
          echo "Skipping file $FILE: unknown MIME type $MIME" >> "$DIR/skipped_files"
          SIZE=0 # don't count the size to collected files
          ;;
      esac
}

# add files (from newest to oldest) that are non zero, readable, regular file or symlink, until $MAXSIZE limit is reached
add_files() {
  SUMSIZE=0
  # sort regular+symlink readable nonempty files per modification time, newest first - assuming no space in a filename
  for FILE in $(find -L $* -type f -readable -size +0b -printf "%T@ %p\n" 2> /dev/null | sort -nr | cut -d' ' -f2 2> /dev/null); do
    # if we are over size limit, skip rest older files
    if [ \( $SUMSIZE -gt $MAXSIZE \) -a \( $MAXSIZE -gt 0 \) ]; then
      printv " - $FILE (skipped due to size)"
    else
      printv " - $FILE"
      SUBDIR=$(dirname $FILE)
      [ ! -d "$DIR$SUBDIR" ] && mkdir -p "$DIR$SUBDIR"
      # if the file is symlink, copy source and add target
      if [ -h $FILE ]; then
        cp -a "$FILE" "$DIR$FILE"
        FILE=$(readlink -f $FILE)
      fi
      add_file $FILE
      SUMSIZE=$(($((SUMSIZE))+$((SIZE))))
    fi
    touch -c -r "$SUBDIR" "$DIR$SUBDIR"
  done
}

# default values
DIR=""
NOGENERIC=0
NOTAR=0
MAXSIZE=10485760   # 10 MB in bytes
COMPRESS=""
PRINTPASS=0
QUIET=0
VERBOSE=0
DEBUG=0

if type -p xz >/dev/null; then
  COMPRESS="xz -1"
  EXTENSION=".xz"
elif type -p bzip2 >/dev/null; then
  COMPRESS="bzip2 -1"
  EXTENSION=".bz2"
elif type -p gzip >/dev/null; then
  COMPRESS="gzip -5"
  EXTENSION=".gz"
else
  COMPRESS="cat"
  EXTENSION=""
fi

# read optional configuration file with user-defined defaults
CONF_FILE=/usr/share/foreman/config/foreman-debug.conf
test -f $CONF_FILE && source $CONF_FILE

while getopts "d:gam:s:j:uqpvhx" opt; do
  case $opt in
    d)
      DIR="$OPTARG"
      ;;
    g)
      NOGENERIC=1
      ;;
    a)
      NOTAR=1
      ;;
    p)
      PRINTPASS=1
      ;;
    q)
      QUIET=1
      ;;
    v)
      VERBOSE=1
      ;;
    m)
      error "Warning: -m option has no effect, use -s option"
      ;;
    j)
      COMPRESS="$OPTARG"
      EXTENSION=".$(echo "$OPTARG" | awk '{ print $1 }')"
      ;;
    s)
      #read the value and convert from MB to bytes
      MAXSIZE="$OPTARG"
      MAXSIZE=$((MAXSIZE*1024*1024))
      ;;
    x)
      # this option is not docummented - use for extra output,
      # skip slow items and to disable root check
      DEBUG=1
      ;;
    h)
      usage
      exit
      ;;
    ?)
      error "Invalid option: $OPTARG"
      usage
      exit
      ;;
  esac
done

[ $DEBUG -eq 0 -a $EUID -ne 0 ] && error "This script must be run as root" && exit 1

# some tasks take long time, print a banner (unless quiet mode was selected)
qprintf "Processing... (takes a while)\n"

# determine distribution family
if [ -f /etc/debian_version ]; then
    OS=debian
    OS_RELEASE=$(head -n1 /etc/debian_version)
elif [ -f /etc/redhat-release ]; then
    OS=redhat
    OS_RELEASE=$(head -n1 /etc/redhat-release)
elif type -p lsb_release >/dev/null; then
    OS=$(lsb_release -si 2>/dev/null)
    OS_RELEASE=$(lsb_release -sr 2>/dev/null)
elif type -p rpm >/dev/null; then
    OS=$(rpm -q --whatprovides redhat-release --queryformat '%{NAME}')
    OS_RELEASE=$(rpm -q --whatprovides redhat-release --queryformat '%{VERSION}')
else
    OS=$(uname -s)
    OS_RELEASE="Unknown"
fi
printv "Determined $OS distribution"

clean_temp() {
  # prevent from removing anything user-defined
  if [[ "$NOTAR" -eq 0 && "$DIR" == /var/tmp* ]]; then
    printv "Cleaning $DIR"
    rm -rf "$DIR"
  fi
  if [[ "$TMPDIR" == /var/tmp* ]]; then
    printv "Cleaning $TMPDIR"
    rm -rf "$TMPDIR"
  fi
}

install_clean_trap() {
  trap clean_temp EXIT
}

if [ -z "$DIR" ]; then
  DIR=$(mktemp -d $(basename $0)-XXXXX -p /var/tmp)
else
  [ ! -d "$DIR" ] && mkdir -p "$DIR"
fi
export TMPDIR=$(mktemp -d foreman-debug-auxtmp-XXXXX -p /var/tmp)
install_clean_trap
printv "Created $DIR and $TMPDIR"

TARBALL="$DIR.tar$EXTENSION"

# GENERIC ARTIFACTS

if [ $NOGENERIC -eq 0 ]; then
  printv "Collecting generic system information"
  add_cmd "date" "date"
  add_cmd "lsb_release -a" "lsb_release"
  add_cmd "uname -a" "uname"
  add_cmd "cat /proc/cpuinfo" "cpuinfo"
  add_cmd "cat /proc/meminfo" "meminfo"
  add_cmd "ulimit -a" "ulimit"
  add_cmd "lsmod" "lsmod"
  add_cmd "iptables -L -v -n" "iptables"
  add_cmd "ifconfig -a" "ifconfig"
  add_cmd "route -n" "route"
  add_cmd "netstat -putna" "netstat"
  add_cmd "ip a" "ip_a"
  add_cmd "ip r" "ip_r"
  add_cmd "ss -putna" "ss"
  add_cmd "cat /etc/hosts" "hosts"
  add_cmd "ping -c1 -W1 localhost" "ping_localhost"
  add_cmd "ping -c1 -W1 $(hostname)" "ping_hostname"
  add_cmd "ping -c1 -W1 $(hostname -f)" "ping_hostname_full"
  add_cmd "host $(hostname -f)" "hostname_dns_check"
  type scl &>/dev/null && \
    add_cmd "scl -l" "software_collections"

  add_cmd "ps auxwwwZ" "process_list"
  add_files /var/log/messages*
  add_files /var/log/audit/audit.log*
  add_files /var/log/syslog*
  add_cmd "getenforce" "selinux_state"
  add_cmd "ausearch -m AVC -m USER_AVC -m SELINUX_ERR | head -n 100" "selinux_first_denials.log"
  add_cmd "ausearch -m AVC -m USER_AVC -m SELINUX_ERR || grep AVC /var/log/audit/audit.log" "selinux_denials.log"
  if [ -f /usr/sbin/selinuxenabled ] && /usr/sbin/selinuxenabled; then
    [[ -f /var/lib/sepolgen/interface_info ]] || sepolgen-ifgen &>/dev/null
    add_cmd "audit2allow -Ra || audit2allow -a" "selinux_audit2allow"
    add_cmd "semodule -l" "selinux_modules"
    add_cmd "semanage boolean -l" "selinux_booleans"
    add_cmd "semanage fcontext -l" "selinux_fcontext"
  fi

  if [ "$OS" = "redhat" ]; then
    [ $DEBUG -eq 0 ] && add_cmd "rpm -qa" "installed_packages"
  elif [ "$OS" = "debian" ]; then
    [ $DEBUG -eq 0 ] && add_cmd "dpkg --list" "installed_packages"
  fi

  add_cmd "virt-who -dop" "virt_who"
fi

# FOREMAN RELATED ARTIFACTS

printv "Collecting Foreman-related information"
add_cmd "rpm -qa '*foreman*' || dpkg -l '*foreman*' | sort" "foreman_packages"
add_cmd "ruby --version" "version_ruby"
add_cmd "puppet --version" "version_puppet"
add_cmd "gem list" "gem_list"
add_cmd "scl enable $SCLNAME 'gem list'" "gem_list_scl"
add_cmd "bundle --local --gemfile=/usr/share/foreman/Gemfile" "bundle_list"
add_cmd "facter" "facts"

add_files /var/log/foreman/apipie_cache*.log*
add_files /var/log/foreman/cron*.log*
add_files /var/log/foreman/db_migrate*.log*
add_files /var/log/foreman/db_seed*.log*
add_files /var/log/foreman/production.log-*
add_files /var/log/foreman/production.log
add_files /var/log/foreman/dynflow_executor.output*
add_files /var/log/foreman-proxy-certs-generate.*log

# Dynflow Sidekiq
add_files /etc/foreman/dynflow/*
add_cmd "systemctl list-units dynflow*" "dynflow_units"
add_cmd "journalctl -u dynflow-sidekiq@*" "dynflow_sidekiq_logs"
add_cmd 'systemctl status "system-dynflow\x2dsidekiq.slice"' "dynflow_sidekiq_status"

# exclude *key.pem files and encryption_key.rb
add_files /etc/foreman/*.{yml,yaml,conf} /etc/foreman/plugins/*.yaml

add_files /etc/foreman-installer/scenarios.d/{*,*/*,*/.*}
add_files /var/log/foreman-installer/
add_files /var/log/foreman-maintain/
add_files /etc/foreman-installer/custom-hiera.yaml
add_files /var/log/foreman-selinux-install.log

add_files /usr/share/foreman/Gemfile*
add_cmd "virsh list" "virsh_list"
add_files /etc/libvirt/* /etc/libvirt/storage/* /etc/libvirt/qemu/* /etc/libvirt/qemu/networks
add_files /var/lib/pgsql/data/*.conf
add_files /var/lib/puppet/ssl/certs/$(hostname -f).pem /var/lib/puppet/ssl/certs/ca.pem
add_files /etc/{httpd,apache2}/conf/*
add_files /etc/{httpd,apache2}/conf.d/*
add_files /etc/{httpd,apache2}/conf.d/*/*
add_files /var/log/{httpd,apache2}/*error_log*
add_files /var/log/{httpd,apache2}/foreman-ssl_access_ssl.log*
add_cmd "echo \"select id,name,value from settings where name not similar to '%(pass|key|secret)'\" | su postgres -c 'psql foreman'" "foreman_settings_table"
add_cmd "echo 'select type,name,host,port,account,base_dn,attr_login,onthefly_register,tls from auth_sources' | su postgres -c 'psql foreman'" "foreman_auth_table"
add_cmd "foreman-selinux-relabel -nv" "foreman_filecontexts"

add_files /etc/{sysconfig,default}/foreman
add_files /etc/{sysconfig,default}/libvirt*
add_files /etc/sysconfig/pgsql
add_files /var/lib/pgsql/data/pg_log/*
add_cmd "passenger-status --show=pool" "passenger_status_pool"
add_cmd "passenger-status --show=requests" "passenger_status_requests"
add_cmd "passenger-status --show=backtraces" "passenger_status_backtraces"
add_cmd "passenger-memory-stats" "passenger_memory"

# Look for any debug extensions provided by plugins
if [ -d "/usr/share/foreman/script/foreman-debug.d" ]; then
  for extension in /usr/share/foreman/script/foreman-debug.d/* ; do
    if [ -x "$extension" ]; then
      printv "Processing extension $extension"
      source "$extension" 2>/dev/null
      # install cleanup trap again just in case it got overwritten
      install_clean_trap
    fi
  done
fi

qprintf "\n\n"
qprintf "%10s %s\n" "HOSTNAME:" "$(hostname -f 2>/dev/null)"
qprintf "%10s %s\n" "OS:" "$OS"
qprintf "%10s %s\n" "RELEASE:" "$OS_RELEASE"
qprintf "%10s %s\n" "FOREMAN:" "$(cat /usr/share/foreman/VERSION 2>/dev/null)"
qprintf "%10s %s\n" "RUBY:" "$(ruby --version 2>/dev/null)"
qprintf "%10s %s\n" "PUPPET:" "$(puppet --version 2>/dev/null)"
test -f /var/log/audit/audit.log && \
  qprintf "%10s %s\n" "DENIALS:" "$(ausearch -m AVC -r | wc -l)"
qprintf "\n\n"

if [ "$NOTAR" -eq 0 ]; then
  pushd "$DIR" >/dev/null
  printv "Compressing directory structure"
  tar -c ../$(basename $DIR) 2>/dev/null | $COMPRESS > "$TARBALL"
  popd >/dev/null
  qprintf "%s: %s\n\n" "A debug file has been created" "$TARBALL ($(stat -c %s "$TARBALL") bytes)"
else
  qprintf "%s: %s\n\n" "A debug directory has been created" "$DIR"
fi

printv "Finished in $SECONDS seconds"

exit 0