Skip to content

Commit

Permalink
Work in progress
Browse files Browse the repository at this point in the history
  • Loading branch information
amousset committed Jun 5, 2019
1 parent 2adcbe5 commit cf0aaa1
Show file tree
Hide file tree
Showing 10 changed files with 145 additions and 36 deletions.
12 changes: 12 additions & 0 deletions qa-test
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

set -ex

# Shellcheck
find . -path ./.git -prune -o -type f -exec grep -Eq '^#!(.*/|.*env +)(sh|bash|ksh)' {} \; -print |
while IFS="" read -r file
do
# with recent shellcheck, "-S error" replaces this hack
shellcheck -f gcc "$file" | grep " error: " && exit 1 || true
done

5 changes: 1 addition & 4 deletions share/commands/agent-log
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ UPDATE_OPTIONS=""
BASE_PATH="/var/rudder/cfengine-community/outputs"
LOGFILE="${BASE_PATH}/previous"

while getopts "wrRmcl:n:d:" opt; do
while getopts "wrRcl:n:d:" opt; do
case $opt in
w)
FULL_STRINGS=1
Expand All @@ -44,9 +44,6 @@ while getopts "wrRmcl:n:d:" opt; do
R)
PRETTY="cat"
;;
m)
MULTILINE=1
;;
l)
LOGFILE="${OPTARG}"
;;
Expand Down
68 changes: 55 additions & 13 deletions share/commands/agent-run
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
# @man +
# @man *-R*: run the agent in completely unparsed mode, with no return code of 1 in case of error. A little faster.
# @man +
# @man *-N-: do not write log in outputs dir (used when called internally)
# @man +
# @man *-b*: run the agent on a specific bundle, this is a debug command that should generally not be used
# @man +
# @man *-D*: define a class for this run
Expand All @@ -35,6 +37,9 @@

. "${BASEDIR}/../lib/common.sh"
. "${BASEDIR}/../lib/cfengine_parsing.sh"
. "${BASEDIR}/../lib/report.sh"

REPORTING_PROTOCOL=$(rudder_json_value 'REPORTING_PROTOCOL')

UPDATE=false
UPDATE_OPTIONS=""
Expand All @@ -43,14 +48,20 @@ CLASSES=""
# Ignore disable-agent flag
FORCE=0
PARTIAL_RUN=0
OUTPUT_LOG=1

# By default, not to mess log file
COLOR="-Cnever"

# When debugging, reports will be huge, skip HTTP reporting
DONT_DO_HTTP=0

while getopts "uiIvdqgwrRmcTb:D:f" opt; do
while getopts "uiIvdqgwrRNcTb:D:f" opt; do
case $opt in
u)
UPDATE=true
;;
i|I)
VERBOSITY="-I ${INFO_CLASS}"
DISPLAY_INFO=1
QUIET=0
UPDATE_OPTIONS="${UPDATE_OPTIONS} -i"
Expand All @@ -60,15 +71,18 @@ while getopts "uiIvdqgwrRmcTb:D:f" opt; do
DISPLAY_INFO=1
QUIET=0
UPDATE_OPTIONS="${UPDATE_OPTIONS} -v"
COLOR="-Calways"
DONT_DO_HTTP=1
;;
d)
VERBOSITY="-d ${DEBUG_CLASS}"
DISPLAY_INFO=1
QUIET=0
UPDATE_OPTIONS="${UPDATE_OPTIONS} -d"
COLOR="-Calways"
DONT_DO_HTTP=1
;;
q)
VERBOSITY=""
DISPLAY_INFO=0
QUIET=1
UPDATE_OPTIONS="${UPDATE_OPTIONS} -d"
Expand All @@ -94,8 +108,8 @@ while getopts "uiIvdqgwrRmcTb:D:f" opt; do
R)
PRETTY="cat"
;;
m)
MULTILINE=1
N)
OUTPUT_LOG=0
;;
b)
BUNDLE="-b ${OPTARG}"
Expand Down Expand Up @@ -135,14 +149,42 @@ if [ "${UPDATE}" = "true" ]; then
code1=$?
fi

# keep same name structure as cf-execd
logfile=$(echo "cf_$(get_hostname)__$(date +%s)_$(LANG=C date +"%a %b %e %H %M %S %Y")_0" | sed 's/[^a-zA-Z0-9]/_/g')
logdir=/var/rudder/cfengine-community/outputs
touch ${logdir}/${logfile}
chmod 600 ${logdir}/${logfile}
"${RUDDER_VAR}/cfengine-community/bin/cf-agent" ${VERBOSITY} ${COLOR} -K ${BUNDLE} ${CLASS} | tee ${logdir}/${logfile} | eval ${PRETTY}
ln -sf ${logdir}/${logfile} ${logdir}/previous
# merge exit codes (this is the eval exit code ... POSIX ...)
# If not launched from cf-execd, we need to take care of logging in outputs
if [ ${OUTPUT_LOG} -eq 1 ]; then
# keep same name structure as cf-execd
logfile=$(echo "cf_$(get_hostname)__$(date +%s)_$(LANG=C date +"%a %b %e %H %M %S %Y")_0" | sed 's/[^a-zA-Z0-9]/_/g')
logdir=/var/rudder/cfengine-community/outputs
touch "${logdir}/${logfile}"
chmod 600 "${logdir}/${logfile}"
log_outputs='tee ${logdir}/${logfile}'
else
log_outputs='cat'
fi

# If reporting in HTTP, we need to take care of timestamped logging
if [ "${REPORTING_PROTOCOL}" = "HTTPS" ] && [ ${DONT_DO_HTTP} -eq 0 ]; then
timestamp="${BASEDIR}/../lib/timestamp.sh"
report_file_name=$(/bin/date -u "+%Y-%m-%dT%T+00:00")@${UUID}.log
report_file="${TMP_REPORTS_DIR}/${report_file_name}"
touch "${report_file}"
chmod 600 "${report_file}"
runlog_output='tee ${report_file}'
else
timestamp='cat'
runlog_output='cat'
fi

"${RUDDER_VAR}/cfengine-community/bin/cf-agent" ${VERBOSITY} ${COLOR} -K ${BUNDLE} ${CLASS} | eval $log_outputs | eval $timestamp | eval $runlog_output | eval ${PRETTY}
code2=$?

if [ ${OUTPUT_LOG} -eq 1 ]; then
ln -sf ${logdir}/${logfile} ${logdir}/previous
fi

if [ "${REPORTING_PROTOCOL}" = "HTTPS" ] && [ ${DONT_DO_HTTP} -eq 0 ]; then
compress_and_sign "${report_file_name}"
fi

# merge exit codes (this is the eval exit code ... POSIX ...)
[ $code1 -ne 0 ] && exit $code1
exit $code2
1 change: 0 additions & 1 deletion share/commands/agent-update
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ QUIET=false
while getopts "iIvdqcf" opt; do
case $opt in
i|I)
VERBOSITY="-I ${INFO_CLASS}"
QUIET=false
;;
v)
Expand Down
7 changes: 1 addition & 6 deletions share/commands/remote-run
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,9 @@ LOOP="xargs -I{}"
MULTIHOST=1
TIMEOUT="-t 300"

while getopts "iIvdqwrRmcTD:j:g:at:u:C:" opt; do
while getopts "iIvdqwrRcTD:j:g:at:u:C:" opt; do
case $opt in
i|I)
VERBOSITY="-I ${INFO_CLASS}"
DISPLAY_INFO=1
QUIET=0
;;
Expand All @@ -69,7 +68,6 @@ while getopts "iIvdqwrRmcTD:j:g:at:u:C:" opt; do
QUIET=0
;;
q)
VERBOSITY=""
DISPLAY_INFO=0
QUIET=1
;;
Expand All @@ -89,9 +87,6 @@ while getopts "iIvdqwrRmcTD:j:g:at:u:C:" opt; do
R)
PRETTY="cat"
;;
m)
MULTILINE=1
;;
D)
CLASSES="${CLASSES},${OPTARG}"
;;
Expand Down
12 changes: 3 additions & 9 deletions share/lib/api_call.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,8 @@
API_URL="https://127.0.0.1/rudder"
TECHNIQUES_DIRECTORY="${CONFIGURATION_DIRECTORY}/technique"

if type curl >/dev/null 2>/dev/null
then
DOWNLOAD_COMMAND="curl --silent --show-error --insecure --location --proxy '' --globoff"
HEADER_OPT="--header"
else
DOWNLOAD_COMMAND="wget --quiet --no-check-certificate --no-proxy -O -"
HEADER_OPT="--header"
fi
DOWNLOAD_COMMAND="${RUDDER_CURL} --silent --show-error --insecure --location --proxy '' --globoff"
HEADER_OPT="--header"

# This functions tests if the API call returns "OK"
simple_api_call() {
Expand Down Expand Up @@ -81,7 +75,7 @@ filtered_api_call() {
action="$3"
filter="$4"
display_command="$5"
curl_command="${DOWNLOAD_COMMAND} -H \"X-API-Token: ${token}\" -X ${action} \"${url}\" ${filter}"
curl_command="${DOWNLOAD_COMMAND} --header \"X-API-Token: ${token}\" --request ${action} \"${url}\" ${filter}"
if ${display_command};
then
printf "${WHITE}${curl_command}${NORMAL}\n\n" >&2
Expand Down
3 changes: 2 additions & 1 deletion share/lib/cfengine_parsing.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ PRETTY_FILTER="${BASEDIR}/../lib/reports.awk"

BUNDLE=""
CLASS=""
VERBOSITY=""
# info as minimal verbosity level for complete reporting
VERBOSITY="-I ${INFO_CLASS}"
# Display logs between Rudder reports
DISPLAY_INFO=0
# Only display a summary at the end of the run, keep the logs unmodified
Expand Down
4 changes: 2 additions & 2 deletions share/lib/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,8 @@ fi
RUDDER_VAR="/var/rudder"

# Standard classes for verbosity
DEBUG_CLASS="-D trace,debug,info"
VERBOSE_CLASS="-D debug,info"
DEBUG_CLASS="-D trace"
VERBOSE_CLASS="-D debug"
INFO_CLASS="-D info"

# Default port for bootstrap
Expand Down
61 changes: 61 additions & 0 deletions share/lib/report.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# signs reports

if [ -f "${RUDDER_JSON}" ]; then
DAVUSER=$(rudder_json_value 'DAVUSER')
DAVPW=$(rudder_json_value 'DAVPASSWORD')
fi

SERVER=$(cat "${RUDDER_VAR}/cfengine-community/policy_server.dat")
TMP_REPORTS_DIR="/var/rudder/tmp/reports/"
REPORTS_DIR="/var/rudder/reports/ready/"

mkdir -p "${TMP_REPORTS_DIR}"
mkdir -p "${REPORTS_DIR}"

# The key to use for signature
PRIVKEY="/var/rudder/cfengine-community/ppkeys/localhost.priv"
CERT="/opt/rudder/etc/ssl/agent.cert"

# Private key passphrase
PASSPHRASE="Cfengine passphrase"

# Should be called on the output file from temp directory
# Everything in the ready dir should be ready to be sent
compress_and_sign() {
# filename
file="$1"
tmp_file="${TMP_REPORTS_DIR}/${file}"
ready_file="${REPORTS_DIR}/${file}.gz"

# We do not include certs as the server already knows them
openssl smime -sign -text -nocerts -signer "${CERT}" -inkey "${PRIVKEY}" -passin "pass:${PASSPHRASE}" \
-in "${tmp_file}" -out "${tmp_file}.signed"
if [ $? -eq 0 ]; then
# Move temp file
mv "${tmp_file}.signed" "${tmp_file}"
else
echo "${red}error${normal}: ${tmp_file} could not be signed"
rm -f "${tmp_file}.signed"
exit 1
fi

gzip -f "${tmp_file}"
if [ $? -ne 0 ]; then
echo "${red}error${normal}: Could not compress ${tmp_file}, exiting"
return 1
fi

# (Very likely) atomic move in ready reports dir
mv "${tmp_file}.gz" "${ready_file}"

# Try to send it.
# If it fails, it will be sent later by the agent
curl --tlsv1.2 --insecure --fail --silent --proxy '' --user "${DAVUSER}:${DAVPW}" --upload-file "${ready_file}" https://${SERVER}/reports/
if [ $? -eq 0 ]; then
# Remove temp file
rm "${ready_file}"
else
# Keep the runlog for future upload by the agent
echo "${yellow}warning${normal}: Could not send ${ready_file}, it will be retried later"
fi
}
8 changes: 8 additions & 0 deletions share/lib/timestamp.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/sh

# Timestamps all lines
while IFS= read -r line; do
# Same as in agent with a T instead of space (iso8601 UTC)
date=$(/bin/date -u "+%Y-%m-%dT%T+00:00")
printf "${date} %s\n" "$line"
done

0 comments on commit cf0aaa1

Please sign in to comment.