rudder-upgrade

#!/bin/bash

set -e

#####################################################################################
# Copyright 2012 Normation SAS
#####################################################################################
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
################################################################################

#####################################################################################
# Upgrade script for Rudder
#####################################################################################
# Currently this script doesn't discriminate on versions, it just always runs
# all the tests and tries to upgrade what it can. It may in the future.
#
# This is mostly OK, because adding unused properties to the
# /opt/rudder/etc/rudder-web.properties configuration file is harmless.
#
# Upgrade checks currently implemented:
# - All versions: upgrade system Techniques automatically and reload the Techniqe library
# - All versions: Check that Rudder database is able to handle backslash
# - 2.4.1 : Add the group serialization table (GroupsNodesJoin) to the database
# - 2.5.0 : Add the automatic reports cleaning properties
# - 2.5.0 : Migration DB schema to add table to the SQL database to store error report logger state
# - 2.5.0 : Add "rudder.batch.reports.logInterval" configuration parameter to rudder-web.properties
# - 2.5.0 : Migration DB schema to add gitcommit table, to link a git commit to a modification
# - 2.5.0 : Migration DB schema to add modificationid column to eventLog table
# - 2.5.0 : Update logback.xml in order to have information about non compliant reports
# - 2.6.0 : Migration LDAP modify entries about System groups
# - 2.6.0 : Migration DB schema to add unexpandedComponentValues column to expectedReports table
# - 2.6.0 : Migration LDAP to unescape entries previously escaped by the user
# - 2.6.0 : Migration SQL to add the necessary workflow tables
# - 2.6.0 : Add the workflows properties and replace the autoDeployOnModification property
# - 2.6.0 : Add modification id to change request table
# - 2.6.0 : Add the property to enable/disable workflow self validation
# - 2.6.0 : Add the property to enable/disable workflow self deployment
# - 2.6.0 : Check the LDAP and PostgreSQL daemons status before upgrading
# - 2.7.0 : Add FileFormat 3 to 4 migration script
# - 2.7.0 : Migration LDAP to add the global parameter OU
# - 2.7.0 : Migration LDAP to add the defaut parameter for file edition
#####################################################################################

# Some variables
VAR_RUDDER="/var/rudder"
TECH_DIR="${VAR_RUDDER}/configuration-repository/techniques"
RUDDER_SHARE=/opt/rudder/share
RUDDER_UPGRADE_TOOLS=${RUDDER_SHARE}/upgrade-tools
LDAP_EXISTS=$(/opt/rudder/sbin/slapcat 2>/dev/null | grep "rudder-configuration" | wc -l)
LDAP_CREDENTIALS=`grep -E "^ldap.(authdn|authpw)=" /opt/rudder/etc/rudder-web.properties | wc -l`
if [ -f /opt/rudder/etc/rudder-web.properties -a ${LDAP_CREDENTIALS} -eq 2 ]; then
	LDAP_USER=$(grep -E "^ldap.authdn=" /opt/rudder/etc/rudder-web.properties |cut -d "=" -f 2-)
	LDAP_PASSWORD=$(grep -E "^ldap.authpw=" /opt/rudder/etc/rudder-web.properties |cut -d "=" -f 2-)
else
	echo "WARNING: LDAP properties are missing in /opt/rudder/etc/rudder-web.properties"
	if [ -f /opt/rudder/etc/openldap/slapd.conf ]; then
		LDAP_USER=$(grep "^rootdn" /opt/rudder/etc/openldap/slapd.conf | sed "s/\w*\s*['\"]\?\([^\"']*\)['\"]\?$/\1/")
		LDAP_PASSWORD=$(grep "^rootpw" /opt/rudder/etc/openldap/slapd.conf | sed "s/\w*\s*['\"]\?\([^\"']*\)['\"]\?$/\1/")
	else
		echo "ERROR: /opt/rudder/etc/openldap/slapd.conf doesn't exist"
		exit 1
	fi
fi
GIT_BRANCH_IS_SET=`grep -E "^rudder.(ptlib|techniqueLibrary).git.refs.path=" /opt/rudder/etc/rudder-web.properties | wc -l`
if [ ${GIT_BRANCH_IS_SET} -eq 1 ]; then
	GIT_BRANCH=$(grep -E "^rudder.(ptlib|techniqueLibrary).git.refs.path=" /opt/rudder/etc/rudder-web.properties |cut -d "=" -f 2- | sed "s@\(refs/heads/\)\?\(refs/tags/\)\?\(refs/remote/origin/\)\?\(.*\)@\4@")
else
	echo "The rudder.ptlib.git.refs.path attribute in rudder-web.properties does not seem to be set"
	echo "Using 'master' by default"
	GIT_BRANCH="master"
fi

# Helper function
# Function to check if a property exists in a configuration file and add it if not
# Parameters:
# - $1 = property name
# - $2 = value to add
function check_and_add_config_property {
    PROPERTY_NAME=$1
    PROPERTY_VALUE=$2
    ATTRIBUTESET=`grep "^${PROPERTY_NAME}[ \t]*=" /opt/rudder/etc/rudder-web.properties | wc -l`
    if [ ${ATTRIBUTESET} -eq 0 ]; then
        echo "${PROPERTY_VALUE}" >> /opt/rudder/etc/rudder-web.properties
        echo "INFO: New configuration property ${PROPERTY_NAME} added to /opt/rudder/etc/rudder-web.properties"
    fi
}

# Helper function
# Function to launch a service status check and retry to poll it in case of a failure
# Parameters:
# - $1 = command to execute
# - $2 = service name
retry_wrapper() {
    tries=0; MAXTRIES=10;
    while [ ${tries} -lt ${MAXTRIES} ]; do
        eval ${1} && RET_SSH=0 || RET_SSH=$?
        # Did we succeed?
        if [ ${RET_SSH} -eq 0 ]; then break; fi
        # OK, we failed
        tries=$((${tries}+1))
        if [ ${tries} -ge ${MAXTRIES} ]; then
            echo "${2} service verification failed after ${MAXTRIES} tries. Aborting the migration."
            echo "Please restart the Rudder service, and start the migration script again:"
            echo "# /etc/init.d/rudder-server-root restart"
            echo "# /opt/rudder/bin/rudder-upgrade"
            /bin/false
        else
            echo -n "."
            /bin/sleep 2
        fi
    done
}

# Before doing anything on git, set the branch to the Technique Reference Library branch
if [ -d /var/rudder/configuration-repository/.git ];then
	cd /var/rudder/configuration-repository/ && git checkout ${GIT_BRANCH} >/dev/null 2>&1
fi

# - 2.6.0 : Check the LDAP and PostgreSQL daemons status before upgrading
if [ ${LDAP_EXISTS} -ne 0 ]; then
  echo -n "INFO: Checking LDAP service status..."
  retry_wrapper '/opt/rudder/bin/ldapsearch -x -D "${LDAP_USER}" -w "${LDAP_PASSWORD}" -s one -b "cn=rudder-configuration" cn >/dev/null 2>&1' 'LDAP'
  echo " OK"
fi

echo -n "INFO: Checking PostgreSQL service status..."
retry_wrapper 'su - postgres -c "psql -t -d rudder -c \"select count(id) from ruddersysevents\"" >/dev/null 2>&1' 'PostgreSQL'
echo " OK"

# Check for configuration property added in 2.5
check_and_add_config_property rudder.batch.reports.logInterval "###############################
# Non compliant reports logger #################################################
###############################

# Rudder can log a line for each 5 minute period when configuration policy is 
# not correctly applied (in error or repaired).
# 
# Default path is /var/log/rudder/compliance/non-compliant-reports.log, and can
# be changed in /opt/rudder/etc/logback.xml.
#
# See online documentation for more details.
#
# This log is generated by a job that runs at a regular interval, by default
# every minute. You can specify this interval (in minutes) below.
# A negative or 0 value disables the job, and won't log any non-compliant reports.
#

rudder.batch.reports.logInterval=1

"
# File Format migration : version 1 (2.3) to 2 (2.4) : Check if an event log has File format 1
# The migration is not supported anymore, explain that you should upgrade first to 2.6 before
RES=$(su - postgres -c "psql -t -d rudder -c \"select count(*) from (select xpath('/entry/*[@fileFormat=1]',data) AS x from eventlog) as Y where array_upper(x, 1) > 0;\"")
RES2=$(su - postgres -c "psql -t -d rudder -c \"select count(*) from (select xpath('/entry/addPending',data) AS x from eventlog) as Y where array_upper(x, 1) > 0;\"")
if [ $RES -ne 0 ] || [ $RES2 -ne 0 ]; then
   echo "WARNING: Some event log are still based on an old file format (file format 1), please upgrade first to 2.6 to make this migration"
fi

# File Format migration : version 2 (2.4) to 3 (2.4) : Check if an event log has File format 2 and set migration flag if needed
RES=$(su - postgres -c "psql -t -d rudder -c \"select count(*) from (select xpath('/entry/*[@fileFormat=2]',data) AS x from eventlog) as Y where array_upper(x, 1) > 0;\"")
RES2=$(su - postgres -c "psql -t -d rudder -c \"select count(*) from (select xpath('/entry/addPending',data) AS x from eventlog) as Y where array_upper(x, 1) > 0;\"")
if [ $RES -ne 0 ] || [ $RES2 -ne 0 ]; then
	psql -q -U rudder -h localhost -d rudder -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-2.4-2.4-set-migration-needed-flag-for-EventLog.sql >/dev/null 2>&1
fi

# 2.7.0: File Format migration from 3 to 4
RES=$(su - postgres -c "psql -t -d rudder -c \"select count(*) from (select xpath('/entry/*[@fileFormat=3]',data) AS x from eventlog) as Y where array_upper(x, 1) > 0;\"")
if [ $RES -ne 0 ]; then
        psql -q -U rudder -h localhost -d rudder -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-2.6-2.7-set-migration-needed-flag-for-EventLog.sql > /dev/null 2>&1
fi

# Upgrade database schema from 2.4 to 2.5 if necessary - first part : Check if the rudder properties table is present, and create it if needed.
RES=$(su - postgres -c "psql -t -d rudder -c \"select count(oid) from pg_class where lower(relname) = 'rudderproperties'\"")
if [ $RES -eq 0 ]; then
	psql -q -U rudder -h localhost -d rudder -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-2.4-2.5-last-error-report-id.sql >/dev/null 2>&1
fi

# Upgrade database schema from 2.4 to 2.5 if necessary - second part : Check if the git commit table is present, and create it if needed.
RES=$(su - postgres -c "psql -t -d rudder -c \"select count(oid) from pg_class where lower(relname) = 'gitcommit'\"")
if [ $RES -eq 0 ]; then
	psql -q -U rudder -h localhost -d rudder -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-2.4-2.5-git-commit.sql >/dev/null 2>&1
fi

# Upgrade database schema from 2.4 to 2.5 if necessary - third part : Check if the modificationId column is present in event log table, and create it if needed.
RES=$(su - postgres -c "psql -t -d rudder -c \"select count(attname) from pg_attribute where attrelid = (select oid from pg_class where relname = 'eventlog') and attname = 'modificationid'\"")
if [ $RES -eq 0 ]; then
	psql -q -U rudder -h localhost -d rudder -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-2.4-2.5-add-modification-id-to-EventLog.sql >/dev/null 2>&1
fi

# Upgrade database schema from 2.5 to 2.6 if necessary - first part : Add a column unexpandedComponentsValues to table ExpectedReports.
RES=$(su - postgres -c "psql -t -d rudder -c \"select count(attname) from pg_attribute where attrelid = (select oid from pg_class where relname = 'expectedreports') and attname = 'unexpandedcomponentsvalues'\"")
if [ $RES -eq 0 ]; then
        psql -q -U rudder -h localhost -d rudder -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-2.5-2.6-unexpanded-value.sql >/dev/null 2>&1
fi

## Change attribute from dn:ruleId=inventory-all,ou=Rules,ou=Rudder,cn=rudder-configuration
## Check if ruleTarget attribute contains all nodes or all except policy server and LDAP is setting up
CHECK_INVENTORY_TARGET=`/opt/rudder/bin/ldapsearch -H ldap://localhost -x -w ${LDAP_PASSWORD} -D ${LDAP_USER} -b "ruleId=inventory-all,ou=Rules,ou=Rudder,cn=rudder-configuration" -LLL "(ruleTarget=*)" >/dev/null 2>&1 | grep "^ruleTarget: special:all$" | wc -l`
if [ ${LDAP_EXISTS} -ne 0 -a ${CHECK_INVENTORY_TARGET} -ne 1 ]
then
  /opt/rudder/bin/ldapmodify -x -D ${LDAP_USER} -w ${LDAP_PASSWORD} -H ldap://localhost  >/dev/null 2>&1 << EOF
dn: ruleId=inventory-all,ou=Rules,ou=Rudder,cn=rudder-configuration
changetype: modify
replace: ruleTarget
ruleTarget: special:all
EOF
fi


# Upgrade system Techniques - always do this!
SRCTECHDIR=/opt/rudder/share/techniques/system/
TRGTECHDIR=/var/rudder/configuration-repository/techniques/system/

if [ -d ${SRCTECHDIR} -a -d ${TRGTECHDIR} ]; then
	if ! diff -Naur /opt/rudder/share/techniques/system/ /var/rudder/configuration-repository/techniques/system/ >/dev/null 2>&1; then
		rsync --delete -rptgoq /opt/rudder/share/techniques/system/ /var/rudder/configuration-repository/techniques/system/
		cd /var/rudder/configuration-repository/techniques/ && git add -A system/ && git commit -m "Upgrade system Techniques - automatically done by rudder-upgrade script" >/dev/null 2>&1
		# For every upgrade, we schedule a Technique reloading REST call on the next CFEngine run
		echo "INFO: A Technique library reload is needed and has been scheduled."
		touch /opt/rudder/etc/force_technique_reload
	fi
fi


# All versions: Check that Rudder database is able to handle backslash
CHECK_BACKSLASH=$(su - postgres -c "psql -t -d rudder -c \"select '\\foo';\"" 2>/dev/null| grep "foo" | wc -l)
if [ ${CHECK_BACKSLASH} -ne 1 ]; then
  echo -n "INFO: Rudder database is not backslash compliant, fixing..."
  su - postgres -c "psql -t -d rudder -c \"alter database rudder set standard_conforming_strings=true;\"" >/dev/null 2>&1
  echo " Done"
  echo -n "INFO: PostgreSQL and Rudder will be restarted..."
  /etc/init.d/postgresql restart >/dev/null 2>&1
  /etc/init.d/jetty restart >/dev/null 2>&1
  echo " Done"
fi


# - 2.5.0 : Update logback.xml in order to have information about non compliant reports
if ! cat /opt/rudder/etc/logback.xml | perl -p0e 's/\n//g' | perl -p0e 's/<!--.(?:(?!-->).)*-->//g' | perl -p0e 's/> *</></g' | grep -E '<property name="REPORT_DIR" value="[^"]+" />' >/dev/null 2>&1
then
    sed -i 's%^ *</configuration>%  <!-- Here come non compliant reports logger -->\n\n  <property name="REPORT_DIR" value="/var/log/rudder/compliance" />\n\n  <!--\n    A file log appender for exploitation logs about failure reports.\n  -->\n  <appender name="REPORTLOG" class="ch.qos.logback.core.FileAppender">\n    <file>${REPORT_DIR}/non-compliant-reports.log</file>\n    <append>true</append>\n    <encoder>\n      <pattern>\%msg\%n</pattern>\n    </encoder>\n  </appender>\n\n  <logger name="non-compliant-reports" level="info" additivity="false">\n    <appender-ref ref="REPORTLOG" />\n    <!-- comment the following appender if you dont want to have logs about non compliant reports in both stdout and reportlog -->\n    <appender-ref ref="STDOUT" />\n  </logger>\n\n</configuration>%' /opt/rudder/etc/logback.xml
fi

# - 2.4.1 : Add the group serialization table (GroupsNodesJoin) to the database
RES=$(su - postgres -c "psql -d rudder -t -c \"select count(1) from pg_class where relname = 'groupsnodesjoin'\"")
if [ $RES -eq 0 ]; then
	psql -q -U rudder -h localhost -d rudder -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-2.4-2.5-group-serialisation.sql >/dev/null 2>&1
fi

# - 2.5.0 : Add the automatic reports cleaning properties
# Check for configuration property added in 2.5
check_and_add_config_property rudder.batch.reportscleaner.archive.TTL "###########################
# Automatic reports cleaning ###########################################################
###########################

# Defaults: archive after 30 days, delete after 90 days.

rudder.batch.reportscleaner.archive.TTL=30
rudder.batch.reportscleaner.delete.TTL=90

# Default frequency: daily
rudder.batch.reportscleaner.frequency=daily

# Values  : [0-59]
# Default : 0
rudder.batch.databasecleaner.runtime.minute=0

# Values : [0-23]
# Default : 0
rudder.batch.databasecleaner.runtime.hour=0

# Values : monday | tuesday | wednesday | thursday | friday | saturday | sunday
# Default : sunday
rudder.batch.databasecleaner.runtime.day=sunday"

# - 2.6.0 : Migration LDAP modify entries about System groups
echo -n "INFO: Modifying system group entries in LDAP if necessary..."
# Ensure that LDAP exist before to do any request
if [ ${LDAP_EXISTS} -ne 0 ]; then
/opt/rudder/bin/ldapmodify -x -D ${LDAP_USER} -w ${LDAP_PASSWORD} -H ldap://localhost -f ${RUDDER_UPGRADE_TOOLS}/rudder-upgrade-modify-system-group-entries.ldif >/dev/null 2>&1
fi
echo " Done"

# - 2.6.0 : Migration LDAP to unescape entries previously escaped by the user
if [ ! -f /opt/rudder/etc/escaping_migration_done ]; then
	echo -n "INFO: Converting escaped directive variabled to plain format..."
	# If some entries match something like \" or \\, convert them to either " or \ and change them in the LDAP.
	/opt/rudder/bin/ldapsearch -H ldap://localhost -x -D ${LDAP_USER} -w ${LDAP_PASSWORD} -b 'techniqueCategoryId=Active Techniques,ou=Rudder,cn=rudder-configuration' -LLL '(&(objectClass=directive)(|(directiveVariable=*\\"*)(directiveVariable=*\\\\*)))' directiveVariable 2>/dev/null \
	| perl -p0e "s/\n //g" \
	|sed "s/\(ou=Rudder,cn=rudder-configuration\)/\1\nchangetype: modify\nreplace: directiveVariable/" \
	|sed "s/\\\\\"/\"/g" \
	|sed "s%\\\\\\\%\\\%g" \
	|/opt/rudder/bin/ldapmodify -H ldap://localhost -x -D ${LDAP_USER} -w ${LDAP_PASSWORD} >/dev/null 2>&1

	touch /opt/rudder/etc/escaping_migration_done
	echo "Done"
fi

# - 2.6.0 : Migration SQL to add the necessary workflow tables
RES=$(su - postgres -c "psql -d rudder -t -c \"select count(1) from pg_class where relname = 'changerequest'\"")
if [ $RES -eq 0 ]; then
	psql -q -U rudder -h localhost -d rudder -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-2.5-2.6-add_workflow_support.sql >/dev/null 2>&1
fi

# - 2.6.0 : Add the workflows properties and replace the autoDeployOnModification property

# Get the old value if applicable
if egrep -q "^rudder.autoDeployOnModification" /opt/rudder/etc/rudder-web.properties; then
	AUTODEPLOY_BOOL=$(grep ^rudder.autoDeployOnModification /opt/rudder/etc/rudder-web.properties|sed "s/.*=\(.*\)/\1/")
fi

# Reverse the value (if autodeploy is enabled, workflows aren't enabled by default)
if [ "z${AUTODEPLOY_BOOL}" = "ztrue" ]; then
	WORKFLOWS_BOOL="false"
elif [ "z${AUTODEPLOY_BOOL}" = "zfalse" ]; then
	WORKFLOWS_BOOL="true"
fi

# Replace the old property if applicable
sed -i "s/^\(rudder.autoDeployOnModification=.*\)$/# This variable has been replaced by\n# rudder.workflow.enabled, see below.\n#\1/" /opt/rudder/etc/rudder-web.properties

# Add the new property according to the old one value
check_and_add_config_property rudder.workflow.enabled "
############
# Workflows ############################################################################
############

# Boolean, defaults to false.
# If true, the Rudder validation workflow will
# be enabled for all configuration changes made
# in the application.
#
rudder.workflow.enabled=${WORKFLOWS_BOOL:-"false"}"

# - 2.6.0 : Add modification id column to change request table
RES=$(su - postgres -c "psql -t -d rudder -c \"select count(attname) from pg_attribute where attrelid = (select oid from pg_class where relname = 'changerequest') and attname = 'modificationid'\"")
if [ $RES -eq 0 ]; then
        psql -q -U rudder -h localhost -d rudder -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-2.6-2.6-add-modification-Id-change-request-column.sql >/dev/null 2>&1
fi

# - 2.6.0 : Add the property to enable/disable workflow self validation
check_and_add_config_property rudder.workflow.self.validation "
# Enable self validation (default false)
#
# If false, no change request can be validated by it's creator
# So every change request will need to be reviewed by someone different from
# who requested this changed
#
# Boolean, defaults to false.

rudder.workflow.self.validation=false"

# - 2.6.0 : Add the property to enable/disable workflow self deployment
check_and_add_config_property rudder.workflow.self.deployment "
# Enable self deployment (default true)
#
# If false, no change request can be deployed by it's creator
# So every change request will need to be deployed by someone different from
# who requested this changed
#
# Boolean, defaults to true.

rudder.workflow.self.deployment=true"

if [ ${LDAP_EXISTS} -ne 0 ]; then
  # - 2.7.0 : Migration LDAP to add the global parameter ou
  LDAP_TEST_GLOBAL_PARAMETER=$(/opt/rudder/sbin/slapcat 2>/dev/null | grep "dn: ou=Parameters,ou=Rudder,cn=rudder-configuration" | wc -l)
  if [ ${LDAP_TEST_GLOBAL_PARAMETER} -eq 0 ]; then
    echo -n "INFO: The Rudder OpenLDAP schema is not up to date, adding Global Parameter ou..."
    /opt/rudder/bin/ldapadd -x -D "${LDAP_USER}" -w "${LDAP_PASSWORD}" -H ldap://localhost -f ${RUDDER_UPGRADE_TOOLS}/rudder-upgrade-LDAP-schema-2.6-2.7-add-global-parameter-ou.ldif >/dev/null 2>&1
    echo " Done."
  fi

  # - 2.7.0 : Migration LDAP to add the default parameter for file edition
  LDAP_TEST_DEFAULT_PARAMETER=$(/opt/rudder/bin/ldapsearch -H ldap://localhost -x -w "${LDAP_PASSWORD}" -D "${LDAP_USER}" -b "parameterName=rudder_file_edit_header,ou=Parameters,ou=Rudder,cn=rudder-configuration"  -LLL parameterName 2>/dev/null | grep "parameterName" | wc -l)
  if [ ${LDAP_TEST_DEFAULT_PARAMETER} -eq 0 ]; then
    echo -n "INFO: Adding default Global Parameter containing the header put in file managed by Rudder..."
    /opt/rudder/bin/ldapadd -x -D "${LDAP_USER}" -w "${LDAP_PASSWORD}" -H ldap://localhost -f ${RUDDER_UPGRADE_TOOLS}/rudder-upgrade-LDAP-schema-2.6-2.7-add-default-global-parameter.ldif >/dev/null 2>&1
    echo " Done."
  fi
fi

# For every upgrade, we force the root server to run a new inventory on the next CFEngine run
touch /opt/rudder/etc/force_inventory