rudder-upgrade

#!/bin/bash

set -eE

function anomaly_handler() {
  echo ""
  echo "ERROR: An error happened during the step: ${STEP} (l $1)"
  echo "ERROR: The migration will now stop"
  echo "ERROR: Please try to run /opt/rudder/bin/rudder-upgrade again"
}

trap 'anomaly_handler ${LINENO}' ERR INT TERM

#####################################################################################
# Copyright 2012 Normation SAS
#####################################################################################
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, Version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
################################################################################

#####################################################################################
# Upgrade script for Rudder
#####################################################################################
# Currently this script doesn't discriminate on versions, it just always runs
# all the tests and tries to upgrade what it can. It may in the future.
#
# This is mostly OK, because adding unused properties to the
# /opt/rudder/etc/rudder-web.properties configuration file is harmless.
#
# Upgrade checks currently implemented:
# - All versions : Check if access credentials in rudder-web.properties match the ones from rudder-passwords.conf
# - All versions : upgrade system Techniques automatically and reload the Technique library
# - All versions : Check that Rudder database is able to handle backslash
# - All versions : Check for the PostgreSQL version
# - All versions : Create ncf technique update flag and restart at the end of the script
# - 5.0          : Automatically upgrade Techniques if autoupdate-technique-library is set
# - 5.0.3        : Change LDAP attribute ipNetworkNumber -> networkSubnet
# - 5.0.5        : Properties related to "incoming inventory watch"
# - 5.0.7        : Tune autovacuum on a table basis
# - 5.0.12       : Change indexes on RudderSysEvents table
# - 6.0.0        : Remove ncf hooks
# - 6.0.0        : Keep syslog as default ptotocol on upgrade
# - 6.0.4        : Ensure that rudder-slapd sockbuf_max_incoming_auth is not lost at upgrade 
# - 6.1.0        : Migrate RudderUniqueID to the template format
# - 6.1.0        : Add indexes cn,softwareVersion,directiveId in slapd.conf 
# - 6.1.1        : Update apache conf if it does not contain technique editor conf (file not updated by packaging
# - 6.2.0~beta2  : Add sofware and modifyTimestamp indexes in slapd.conf
####################################################################################

# Some paths
RUDDER_OPT="/opt/rudder"
RUDDER_VAR="/var/rudder"
CONFIGURATION_REPOSITORY="${RUDDER_VAR}/configuration-repository"
RUDDER_SHARE="${RUDDER_OPT}/share"
RUDDER_NCF_SOURCE_DIRECTORY="/usr/share/ncf"
RUDDER_UPGRADE_TOOLS="${RUDDER_SHARE}/upgrade-tools"

# If true, display a message at the end, asking to re-run the script
INCOMPLETE_MIGRATION=false

# Ensure our PATH includes Rudder's binaries
export PATH=${PATH}:${RUDDER_OPT}/bin

# Set to defined value if not defined by environment
: ${RUDDER_NO_TECHNIQUE_AUTOCOMMIT:=0}

# Get how many access credentials we got for LDAP and SQL in /opt/rudder/etc/rudder-web.properties
# (should have 2 for each, user and password)
LDAP_CREDENTIALS=$(grep -c -E "^ldap.auth(dn|pw)[ \t]*=" /opt/rudder/etc/rudder-web.properties || true)
SQL_CREDENTIALS=$(grep -c -E "^rudder.jdbc.(username|password)[ \t]*=" /opt/rudder/etc/rudder-web.properties || true)

TOTAL_CREDENTIALS=$((LDAP_CREDENTIALS+SQL_CREDENTIALS))

LDAP_SERVER='localhost'
LDAP_PORT='389'
if [ -f /opt/rudder/etc/rudder-web.properties -a ${TOTAL_CREDENTIALS} -eq 4 ]; then
  # Get the database access credentials from the rudder-web.properties file
  LDAP_USER="$(grep -E '^ldap.authdn[ \t]*=' ${RUDDER_OPT}/etc/rudder-web.properties | cut -d "=" -f 2-)"
  LDAP_PASSWORD="$(grep -E '^ldap.authpw[ \t]*=' ${RUDDER_OPT}/etc/rudder-web.properties | cut -d "=" -f 2-)"

  SQL_USER="$(grep -E '^rudder.jdbc.username[ \t]*=' ${RUDDER_OPT}/etc/rudder-web.properties | cut -d "=" -f 2-)"
  SQL_PASSWORD="$(grep -E '^rudder.jdbc.password[ \t]*=' ${RUDDER_OPT}/etc/rudder-web.properties | cut -d "=" -f 2-)"
  SQL_SERVER="$(grep -E '^rudder.jdbc.url[ \t]*=' ${RUDDER_OPT}/etc/rudder-web.properties | cut -d '=' -f 2- | sed 's%^.*://\(.*\):\(.*\)/.*$%\1%')"
  SQL_PORT="$(grep -E '^rudder.jdbc.url[ \t]*=' ${RUDDER_OPT}/etc/rudder-web.properties | cut -d '=' -f 2- | sed 's%^.*://\(.*\):\(.*\)/.*$%\2%')"
  SQL_DATABASE="$(grep -E '^rudder.jdbc.url[ \t]*=' ${RUDDER_OPT}/etc/rudder-web.properties | cut -d '=' -f 2- | sed 's%^.*://.*:.*/\(.*\)$%\1%')"

  export PGPASSWORD="${SQL_PASSWORD}"
else
  # No database access credentials in rudder-web.properties... Try anyway using "guessed" values.
  echo "WARNING: Database access credentials are missing in /opt/rudder/etc/rudder-web.properties, trying to guess adequate values."
  LDAP_USER=$(grep "^rootdn" /opt/rudder/etc/openldap/slapd.conf | sed "s/\w*\s*['\"]\?\([^\"']*\)['\"]\?$/\1/")
  LDAP_PASSWORD=$(grep "^rootpw" /opt/rudder/etc/openldap/slapd.conf | sed "s/\w*\s*['\"]\?\([^\"']*\)['\"]\?$/\1/")

  SQL_USER="rudder"
  SQL_PASSWORD="Normation"
  SQL_SERVER="localhost"
  SQL_PORT="5432"

  # We rely on .pgpass instead
  unset PGPASSWORD
fi

# Override any server values with those from ${RUDDER_ROLES_FILE}, unless this is a monolithic
# Rudder server installation (to keep using localhost)
RUDDER_ROLE_SERVER_ROOT="${RUDDER_OPT}/etc/server-roles.d/rudder-server-root"
RUDDER_ROLES_FILE="${RUDDER_VAR}/cfengine-community/inputs/rudder-server-roles.conf"
if [ ! -f ${RUDDER_ROLE_SERVER_ROOT} -a -f ${RUDDER_ROLES_FILE} ]; then
  ROLE_HOSTNAME=$(grep "^rudder-db:" ${RUDDER_ROLES_FILE} | cut -d: -f2 | cut -d, -f1 | tr -d " ")
  if [ ! -z ${ROLE_HOSTNAME} ]; then
    SQL_SERVER=${ROLE_HOSTNAME}
  fi
fi

# Commands
LDAP_PARAMETERS="-H ldap://${LDAP_SERVER}:${LDAP_PORT}/ -D ${LDAP_USER} -w ${LDAP_PASSWORD} -x"

LDAPSEARCH="ldapsearch ${LDAP_PARAMETERS} -LLL"
LDAPMODIFY="ldapmodify ${LDAP_PARAMETERS}"
LDAPADD="ldapadd ${LDAP_PARAMETERS}"

PSQL="psql -q -h ${SQL_SERVER} -p ${SQL_PORT} -U ${SQL_USER}"

################################################################################
# Helper functions
################################################################################

# Helper function
# Function to check if a property exists in a configuration file and add it if not
# Parameters:
# - $1 = property name
# - $2 = value to add
check_and_add_config_property() {
  PROPERTY_NAME=$1
  PROPERTY_VALUE=$2
  ATTRIBUTESET=$(grep -c "^${PROPERTY_NAME}[ \t]*=" /opt/rudder/etc/rudder-web.properties || true)
  if [ ${ATTRIBUTESET} -eq 0 ]; then
    echo "${PROPERTY_VALUE}" >> /opt/rudder/etc/rudder-web.properties
    echo "INFO: New configuration property ${PROPERTY_NAME} added to /opt/rudder/etc/rudder-web.properties"
  fi
}

# Helper function
# Function to launch a service status check and retry to poll it in case of a failure
# Parameters:
# - $1 = command to execute
# - $2 = service name
retry_wrapper() {
  tries=0; MAXTRIES=10;
  while [ ${tries} -lt ${MAXTRIES} ]; do
    eval ${1} && RET_SSH=0 || RET_SSH=$?
    # Did we succeed?
    if [ ${RET_SSH} -eq 0 ]; then return 0; fi
    # OK, we failed
    tries=$((${tries}+1))
    if [ ${tries} -ge ${MAXTRIES} ]; then
      echo " FAILED" # we are on a line that says "Checking ...", so finish it before printing anything else
      echo "${2} service verification failed after ${MAXTRIES} tries."
      return 1
    else
      echo -n "."
      /bin/sleep 2
    fi
  done
}

# Helper function
# Function to update rudder's configuration repository from a system directory
# Parameters:
# - $1 = origin directory
# - $2 = destination directory (in configuration repository)
update_rudder_repository_from_system_directory() {
  # Prime work variables first
  SRCTECHDIR="${1}"
  TRGTECHDIR="${CONFIGURATION_REPOSITORY}/${2}"

  if [ -d ${SRCTECHDIR} -a -d ${TRGTECHDIR} ]; then
    if ! diff -Naur ${SRCTECHDIR} ${TRGTECHDIR} >/dev/null 2>&1; then
      rsync --delete -rptgoq ${SRCTECHDIR} ${TRGTECHDIR}
      if [ x"${RUDDER_NO_TECHNIQUE_AUTOCOMMIT}" != x"1" ]; then
        cd ${CONFIGURATION_REPOSITORY}/ && git add -A ${2} && git commit -m "Upgrade system Techniques from ${SRCTECHDIR} - automatically done by rudder-upgrade script" >/dev/null 2>&1
        # Schedule a Technique library reload later because of the update
        echo "INFO: A Technique library reload is needed and has been scheduled."
        touch /opt/rudder/etc/force_technique_reload
      else
        echo "WARN: Autocommit of system techniques is disabled."
        echo "Please review and commit changes, then reload technique library on your own!"
      fi
    fi
  fi
}

# Helper function to compare version numbers with 2 components
ver() { printf "1%03d%03d" `echo "$1" | tr '.' ' '`; }


################################################################################
# Update credentials
################################################################################

# - All versions: Check if access credentials in rudder-web.properties match the ones from rudder-passwords.conf
# For every property file, check if the access credentials match between the property file and
# the rudder-passwords.conf file. If not, update the properties.
#
# This is to prevent an upgrade failure if the properties have been resetted during a package upgrade.
update_credentials() {
  if [ -f /opt/rudder/etc/rudder-passwords.conf ]; then

    # Get the passwords stored in rudder-passwords.conf
    REFERENCE_LDAP_PASSWORD=$(grep -E "^RUDDER_OPENLDAP_BIND_PASSWORD:" /opt/rudder/etc/rudder-passwords.conf |cut -d ":" -f 2-)
    REFERENCE_SQL_PASSWORD=$(grep -E "^RUDDER_PSQL_PASSWORD:" /opt/rudder/etc/rudder-passwords.conf |cut -d ":" -f 2-)

    property_file="rudder-web.properties"
    echo -n "INFO: Checking if ${property_file} database access credentials are all right..."

    if [ ! -e /opt/rudder/etc/${property_file} ]; then echo " non existant, skipping"; continue; fi

    # 1 - Check LDAP credentials on all property files

    # Get the LDAP password according to the property file
    PROPERTY_LDAP_PASSWORD=$(grep -E "^ldap.authpw[ \t]*=" /opt/rudder/etc/${property_file} |cut -d "=" -f 2-)
    if [ "z${REFERENCE_LDAP_PASSWORD}" = "z${PROPERTY_LDAP_PASSWORD}" ]; then
      echo -n " LDAP OK, "
    else
      sed -i "s%ldap.authpw[ \t]*=.*%ldap.authpw=${REFERENCE_LDAP_PASSWORD}%" /opt/rudder/etc/${property_file}
      LDAP_PASSWORD=${REFERENCE_LDAP_PASSWORD}
      # Need to restart rudder-slapd so that services can connect to it
      systemctl restart rudder-slapd
      echo -n " LDAP Credentials updated, "
    fi

    # 2 - SQL properties only apply to rudder-web.properties

    # Get the SQL password according to the property file
    PROPERTY_SQL_PASSWORD=$(grep -E "^rudder.jdbc.password[ \t]*=" /opt/rudder/etc/${property_file} |cut -d "=" -f 2-)
    if [ "${REFERENCE_SQL_PASSWORD}" = "${PROPERTY_SQL_PASSWORD}" ]; then
      echo " SQL OK"
    else
      # Credentials from the properties and the rudder-password.conf do not match, update the properties.
      sed -i "s%rudder.jdbc.password[ \t]*=.*%rudder.jdbc.password=${REFERENCE_SQL_PASSWORD}%" /opt/rudder/etc/${property_file}
      SQL_PASSWORD=${REFERENCE_SQL_PASSWORD}
      export PGPASSWORD="${SQL_PASSWORD}"
      echo " SQL Credentials updated"
    fi

  else
    echo "INFO: The /opt/rudder/etc/rudder-passwords.conf file is absent. LDAP and SQL passwords not checked in {rudder,inventory}-web.properties"
  fi
}

################################################################################
# Check git branch
################################################################################

# Before doing anything on git, set the branch to the Technique Reference Library branch
set_git_branch() {
  if [ -d ${CONFIGURATION_REPOSITORY}/.git ];then
    cd ${CONFIGURATION_REPOSITORY}/ && git checkout master >/dev/null 2>&1
  fi
}

################################################################################
# Update ncf and system techniques
################################################################################

# Upgrade system techniques from system directory
upgrade_system_techniques() {
  STEP="Upgrade system Techniques"

  update_rudder_repository_from_system_directory /opt/rudder/share/techniques/system/common/ techniques/system/common/
  update_rudder_repository_from_system_directory /opt/rudder/share/techniques/system/distributePolicy/ techniques/system/distributePolicy/
  update_rudder_repository_from_system_directory /opt/rudder/share/techniques/system/inventory/ techniques/system/inventory/
  update_rudder_repository_from_system_directory /opt/rudder/share/techniques/system/server-roles/ techniques/system/server-roles/
}

# Upgrade techniques
upgrade_techniques() {
  STEP="Upgrade Techniques"

  /opt/rudder/bin/rudder server upgrade-techniques  --autoupdate-technique-library --during-upgrade

  # Migrate RudderUniqueID to the template format
  NB_RUDDER_UNIQUE_ID=$((grep -Er --exclude="*.xml"  "[^\&]RudderUniqueID" /var/rudder/configuration-repository/techniques 2>/dev/null || true) | wc -l) 
  if [ ${NB_RUDDER_UNIQUE_ID} -ne 0 ]; then
    echo -n "INFO: Enforcing that Techniques are using correct format" 
    find ${CONFIGURATION_REPOSITORY}/techniques -type f \( ! -iname '*.xml' \) -print0 | xargs -0 sed -i "s/\([^&]\|^\)RudderUniqueID\([^&]\|$\)/\1\&RudderUniqueID\&\2/g"
    cd ${CONFIGURATION_REPOSITORY}/techniques && git add . >/dev/null 2>&1
    git commit -m "Upgrade Techniques to template format - automatically done by rudder-upgrade script" >/dev/null 2>&1
    # Schedule a Technique library reload later because of the update
    touch /opt/rudder/etc/force_technique_reload
    echo " Done"
  fi
}

# Upgrade ncf from ncf package
upgrade_ncf() {
  # All versions: Upgrade ncf
  STEP="Upgrade ncf"

  mkdir -p ${RUDDER_VAR}/ncf/common
  if [ -d ${RUDDER_NCF_SOURCE_DIRECTORY}/tree ]; then
    if ! diff -aur ${RUDDER_NCF_SOURCE_DIRECTORY}/tree/ ${RUDDER_VAR}/ncf/common/ >/dev/null 2>&1; then
      rsync --delete -rpgoq ${RUDDER_NCF_SOURCE_DIRECTORY}/tree/ ${RUDDER_VAR}/ncf/common/
    fi
  fi

  # Security check: Ensure techniques written from the Technique Editor are not world-readable
  if [ -x ${CONFIGURATION_REPOSITORY}/ncf/50_techniques ]; then
    chmod -R o-rwx ${CONFIGURATION_REPOSITORY}/ncf/50_techniques/
  fi


  # 5.1.0: Remove ncf.conf file
  if [ -f ${CONFIGURATION_REPOSITORY}/ncf/ncf.conf ]; then
    current_path=$(pwd)
    cd ${CONFIGURATION_REPOSITORY}
    git rm -f ncf/ncf.conf
    git commit --allow-empty -m "Remove ncf.conf from configuration repository"
    cd ${current_path}
  fi

  # 5.1.0: Remove ncf hooks directory
  if [ -d ${CONFIGURATION_REPOSITORY}/ncf/ncf-hooks.d ]; then
    CURRENT="$(pwd)"
    cd ${CONFIGURATION_REPOSITORY}/ncf
    git rm -rf ncf-hooks.d
    git commit --allow-empty -m "Remove ncf-hooks.d from configuration repository"
    cd ${CURRENT}
  fi

  # All Versions: Touch ncf_update_flag_file jetty will be restarted at the end of the script 
  touch /opt/rudder/etc/force_ncf_technique_update

  # Handle the case where we have an existing local method and a common method with the same name
  # Display a warning and delete local method
  if [ -x ${CONFIGURATION_REPOSITORY}/ncf/30_generic_methods ]; then
    CURRENT="$(pwd)"
    cd "${CONFIGURATION_REPOSITORY}/ncf/30_generic_methods/"
    commit_needed=0
    for method in *.cf; do
      [ -f "${method}" ] || break
      if [ -f "${RUDDER_NCF_SOURCE_DIRECTORY}/tree/30_generic_methods/${method}" ]; then
        if git ls-files --error-unmatch ${method} >/dev/null 2>&1; then
          git rm "${method}" >/dev/null
          commit_needed=1
          echo "WARNING: The ${method} generic method was already present in local configuration, removing it."
        else
          mv "${method}" "${method}.disabled"
          echo "WARNING: The ${method} generic method was already present in local configuration and not in local repo, disabling it (renamed to ${method}.disabled)."
        fi
        rm -f ${RUDDER_VAR}/ncf/local/30_generic_methods/${method}
      fi
    done
    [ ${commit_needed} -eq 1 ] && git commit --allow-empty -q -m "Remove duplicate local methods - automatically done by rudder-upgrade script"
    cd ${CURRENT}
  fi


}

# Upgrade masterfiles
upgrade_masterfiles() {
  # We should have a masterfiles initialized from initial promises
  STEP="Upgrade masterfiles from initial promises"
  if [ -d "${RUDDER_SHARE}/initial-promises/" ]
  then
    mkdir -p "${RUDDER_VAR}/cfengine-community/masterfiles/"
    rsync --delete -rptgoq "${RUDDER_SHARE}/initial-promises/" "${RUDDER_VAR}/cfengine-community/masterfiles/"
  fi
}

################################################################################
# Check and update config
################################################################################

# Upgrade the file rudder-web.properties
upgrade_rudder_web_properties() {
# 5.0.5: 
# Add inventories.watcher.enable, inventories.root.directory and inventories.watcher.waitForSignatureDuration
  check_and_add_config_property inventories.watcher.enable "
#
# Rudder 5.0.5 introduce a file watcher which react to new
# inventories put in \${inventories.root.directory}/incoming.
# You can (should) use the watcher by setting the property
# 'inventories.watcher.enable' to true.
# 'inventories.watcher.waitForSignatureDuration' is the time
# in seconds the watcher will wait for the signature file
# in case there is only an inventory file before sending
# only inventory to backend.
# You can start (resp. stop) the watcher with POST to API
# /api/watcher/start (resp /api/watcher/stop).
#
inventories.watcher.enable=true
inventories.root.directory=/var/rudder/inventories
inventories.watcher.waitForSignatureDuration=10
"
}

################################################################################
# Check and update database
################################################################################

# Upgrade the eventlog
# This has been separated from the upgrade_database to be run after, as it would allow to create the indexes for eventlog if necessary
# and have only one location for all the eventlog upgrade part
upgrade_eventlog() {
  :
}

# Upgrade the PostgreSQL database
upgrade_database() {
  # All versions: Check that Rudder database is able to handle backslash
  STEP="All versions: Check that Rudder database is able to handle backslash"

  CHECK_BACKSLASH=$(${PSQL} -t -d ${SQL_DATABASE} -c "select '\\foo';" 2>/dev/null | grep -c "foo" || true)
  if [ ${CHECK_BACKSLASH} -ne 1 ]; then
    echo -n "INFO: Rudder database is not backslash compliant, fixing..."
    ${PSQL} -t -d ${SQL_DATABASE} -c "alter database rudder set standard_conforming_strings=true;" >/dev/null 2>&1
    echo " Done"
  fi

  # - 5.0.7: Tune autovacuum on a table basis
  RES=$(${PSQL} -t -d ${SQL_DATABASE} -c "select reloptions from pg_class where relname='reportsexecution'" | grep -c "autovacuum" || true)
  if [ $RES -eq 0 ]; then
    echo -n "INFO: Adapting autovacuum configuration on a table basis"
    ${PSQL} -d ${SQL_DATABASE} -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-5.0.x-5.0.7-tune-autovacuum_on_table_basis.sql > /dev/null
    echo " Done"
  fi

  # - 5.0.12: change index on ruddersysevents to improve performance 
  RES=$(${PSQL} -t -d ${SQL_DATABASE} -c "select count(oid) from pg_class where lower(relname) = 'endrun_control_idx'")
  if [ $RES -eq 0 ]; then
    echo -n "INFO: Changing index on table RudderSysEvents, this may take several minutes"
    ${PSQL} -d ${SQL_DATABASE} -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-5.0.x-5.0.12-composite-index-on-ruddersysevents.sql > /dev/null
    echo " Done"
  fi

  # - 5.0.13: Remove autovacuum on ruddersysevent table
  RES=$(${PSQL} -t -d ${SQL_DATABASE} -c "select reloptions from pg_class where relname='ruddersysevents'" | grep -c "autovacuum_vacuum_scale_factor" || true)
  if [ $RES -ne 0 ]; then
    echo -n "INFO: Removing autovacuum configuration on ruddersysevents"
    ${PSQL} -d ${SQL_DATABASE} -f ${RUDDER_UPGRADE_TOOLS}/dbMigration-5.0.x-5.0.13-remove-autovacuum-on-ruddersysevents.sql > /dev/null
    echo " Done"
  fi

  # Now check the fileFormat in the eventlog
  upgrade_eventlog

  # - All versions: Check for the PostgreSQL version
  STEP="Check for the current PosgresSQL version"

  # If coming from an old install (upgraded from centos7 or sles11)
  if [ -e /var/lib/pgsql/data/PG_VERSION ]; then
    CLUSTER_VERSION=$(cat /var/lib/pgsql/data/PG_VERSION | cut -d . -f 1-2)
    if [ `ver ${CLUSTER_VERSION}` -lt `ver 9.2` ]; then
      echo "ERROR: Your running PostgreSQL cluster, in version ${CLUSTER_VERSION}, is too old for Rudder 5.0+"
      echo "ERROR: Please manually migrate your data to the newer PostgreSQL version installed with Rudder."
    fi
  fi

  RES=$(psql -V|head -1|cut -d " " -f 3|cut -d . -f 1-2)
  if [ `ver ${RES}` -lt `ver 9.2` ]; then
    echo "ERROR: Your PostgreSQL version, ${RES}, is too old and will fail to work properly with Rudder 5.0+."
    echo "ERROR: Please upgrade to at least 9.2."
  fi
}

################################################################################
# Check and update inventory
################################################################################

# Upgrade the LDAP inventory
upgrade_inventory() {
  # - 4.1.17, 4.3.7, 5.0.3: ipNetworkNumber -> networkSubnet
  STEP="Migrate LDAP attribute ipNetworkNumber in networkInterface"
  # change:
  #  dn: networkInterface=etho0,nodeId=0876521e-3c81-4775-85c7-5dd7f9d5d3da,ou=Nodes,ou=Accepted Inventories,ou=Inventories,cn=rudder-configuration
  #  ipNetworkNumber: 192.268.1.1
  #  ipNetworkNumber: ffff:::1
  #
  # into:
  #  dn: networkInterface=lo,nodeId=0876521e-3c81-4775-85c7-5dd7f9d5d3da,ou=Nodes,ou=Accepted Inventories,ou=Inventories,cn=rudder-configuration
  #  changetype: modify
  #  delete: ipNetworkNumber
  #  -
  #  replace: networkSubnet
  #  networkSubnet: 192.268.1.1
  #  networkSubnet: ffff:::1
  #  -
  #
  LDAP_TEST_MIGRATE_NETWORK=$(${LDAPSEARCH} -b "ou=Inventories,cn=rudder-configuration" -s sub '(&(objectClass=networkInterfaceLogicalElement)(ipNetworkNumber=*))' 1.1 2> /dev/null | grep -c "dn: networkInterface=" || true)
  if [ ${LDAP_TEST_MIGRATE_NETWORK} -ne 0 ]; then
    echo -n "INFO: migrate LDAP attribute ipNetworkNumber -> networkSubnet in networkInterface..."
    MIGRATE_IPNET_FILE="/tmp/rudder-migrate-ipNetworkNumber-network.ldif"
    ${LDAPSEARCH} -o ldif-wrap=no -b "ou=Inventories,cn=rudder-configuration" -s sub '(&(objectClass=networkInterfaceLogicalElement)(ipNetworkNumber=*))' 'ipNetworkNumber' 2> /dev/null > ${MIGRATE_IPNET_FILE}
    sed -i -e "s/ipNetworkNumber/networkSubnet/" ${MIGRATE_IPNET_FILE}
    sed -i -e "s/dn:\(.*\)$/dn:\1\nchangetype:modify\ndelete: ipNetworkNumber\n-\nreplace: networkSubnet/" ${MIGRATE_IPNET_FILE}
    sed -i -e "s/^$/-\n/" ${MIGRATE_IPNET_FILE}
    ${LDAPMODIFY} -f ${MIGRATE_IPNET_FILE}
    echo " OK"
  fi

  STEP="Migrate LDAP attribute ipNetworkNumber in node group query"
  # change:
  #  dn: nodeGroupId=ede838d0-b064-4739-a03e-f6a7651dce7b,groupCategoryId=GroupRoot,ou=Rudder,cn=rudder-configuration
  #  jsonNodeGroupQuery: {"select":"node","composition":"And","where":[{"objectType":"networkInterfaceLogicalElement","attribute":"ipNetworkNumber","comparator":"regex","value":"192.168.42.*"}]}
  #
  # into:
  #  dn: nodeGroupId=ede838d0-b064-4739-a03e-f6a7651dce7b,groupCategoryId=GroupRoot,ou=Rudder,cn=rudder-configuration
  #  changetype: modify
  #  replace: jsonNodeGroupQuery
  #  jsonNodeGroupQuery: {"select":"node","composition":"And","where":[{"objectType":"networkInterfaceLogicalElement","attribute":"networkSubnet","comparator":"regex","value":"192.168.42.*"}]}
  #  -
  #
  LDAP_TEST_MIGRATE_GROUP=$(${LDAPSEARCH} -b "ou=Rudder,cn=rudder-configuration" -s sub '(&(objectClass=nodeGroup)(jsonNodeGroupQuery=*ipNetworkNumber*))' 1.1 2> /dev/null | grep -c "dn: nodeGroupId=" || true)
  if [ ${LDAP_TEST_MIGRATE_GROUP} -ne 0 ]; then
    echo -n "INFO: migrate LDAP attribute ipNetworkNumber -> networkSubnet in node groups..."
    MIGRATE_IPNET_FILE="/tmp/rudder-migrate-ipNetworkNumber-group.ldif"
    ${LDAPSEARCH} -o ldif-wrap=no -b "ou=Rudder,cn=rudder-configuration" -s sub '(&(objectClass=nodeGroup)(jsonNodeGroupQuery=*ipNetworkNumber*))' 'jsonNodeGroupQuery' 2> /dev/null > ${MIGRATE_IPNET_FILE}
    sed -i -e "s/^$/-\n\n/" ${MIGRATE_IPNET_FILE}
    sed -i -e "s/ipNetworkNumber/networkSubnet/" ${MIGRATE_IPNET_FILE}
    sed -i -e "s/jsonNodeGroupQuery/changetype: modify\nreplace: jsonNodeGroupQuery\njsonNodeGroupQuery/" ${MIGRATE_IPNET_FILE}
    ${LDAPMODIFY} -f ${MIGRATE_IPNET_FILE}
    echo " OK"
  fi


  # Pre 6.0 to post 6.0 : keep default protocol to syslog and let the user change it if he wants
  LDAP_TEST_PROTOCOL=$(${LDAPSEARCH} -b "ou=Application Properties,cn=rudder-configuration" "propertyName=rudder_report_protocol_default" | grep -c "dn: propertyName=" || true)
  if [ ${LDAP_TEST_PROTOCOL} -eq 0 ]; then
    ${LDAPADD} <<EOF
dn: propertyName=rudder_report_protocol_default,ou=Application Properties,cn=rudder-configuration
objectClass: property
objectClass: top
propertyName: rudder_report_protocol_default
propertyValue: SYSLOG
EOF
  fi


}

upgrade_apache_conf() {

  # 6.1.1: Update apache conf if it does not contain technique editor conf (file not updated by packaging
  APACHE_FILE="/opt/rudder/etc/rudder-apache-webapp-ssl.conf"
  ALIAS_CHECK=$(grep -c "^Alias /ncf-builder" ${APACHE_FILE} || true)
  if [ ${ALIAS_CHECK} -eq 0 ]; then
    echo "" >> ${APACHE_FILE}
    echo "# Technique editor (aka 'ncf builder')" >> ${APACHE_FILE}
    echo "Alias /ncf-builder /usr/share/ncf/builder" >> ${APACHE_FILE}
    echo "<Directory /usr/share/ncf/builder>" >> ${APACHE_FILE}
    echo "  # Allow access from anybody" >> ${APACHE_FILE}
    echo "  Require all granted" >> ${APACHE_FILE}
    echo "</Directory>" >> ${APACHE_FILE}
    echo "INFO: Technique editor apache conf added to ${APACHE_FILE}"
  fi
}

################################################################################
# Check and upgrade slapd.conf
################################################################################
upgrade_slapdconf() {
  # 6.0.4:
  # ensure sockbuf_max_incoming_auth is set in /opt/rudder/etc/openldap/slapd.conf
  MAX_INCOMING_AUTH_SET=$(grep -c "sockbuf_max_incoming_auth" /opt/rudder/etc/openldap/slapd.conf 2>/dev/null || true)  
  if [ ${MAX_INCOMING_AUTH_SET} -eq 0 ]; then
    echo -n "Setting the sockbuf_max_incoming_auth value for slapd.conf"
    sed -i 's/^\(# This is too little for Rudder, as there are some big entries.*\)/\1\nsockbuf_max_incoming_auth 1000000000/' /opt/rudder/etc/openldap/slapd.conf
    echo " Done"
  fi

  # 6.1.0~beta3:
  # Add indexes cn,softwareVersion,directiveId in slapd.conf
  SLAPD_CONF="/opt/rudder/etc/openldap/slapd.conf"
  NEED_REINDEX=false
  for i in "directiveId" "softwareVersion" "cn"; do
    line="index\s*${i}\s*eq"
    INDEX_COUNT=$(grep -c "^${line}" ${SLAPD_CONF} 2>/dev/null || true)
    if [ ${INDEX_COUNT} -eq 0 ]; then
      echo "Adding LDAP index on attribute: ${i}"
      sed -i "/index\s*objectClass\s*eq/a index\t${i}\teq" ${SLAPD_CONF}
      NEED_REINDEX=true
    fi
  done

  # 6.2.0~beta2:
  # Add software and modifyTimestamp indexes in slapd.conf
  for i in "software" "modifyTimestamp"; do
    line="index\s*${i}\s*eq"
    INDEX_COUNT=$(grep -c "^${line}" ${SLAPD_CONF} 2>/dev/null || true)
    if [ ${INDEX_COUNT} -eq 0 ]; then
      echo "Adding LDAP index on attribute: ${i}"
      sed -i "/index\s*objectClass\s*eq/a index\t${i}\teq" ${SLAPD_CONF}
      NEED_REINDEX=true
    fi
  done

  # If user choose to replace configuration file, then /opt/rudder/etc/openldap/slapd.conf
  # already contains the index, but nothing reindexed slapd
  # We need to check for existence of slapd.conf.dpkg-old, its age, and if it contains index
  SLAPD_CONF_BKP="${SLAPD_CONF}.dpkg-old"
  if [ -f ${SLAPD_CONF_BKP} ]; then
    # if file exist, and is less than 1 hour old, check for index in it
    if [ $(($(date +%s) - $(stat -c '%Z'  ${SLAPD_CONF_BKP}))) -lt 3600 ]; then
      INDEX_BKP_COUNT=$(grep -c "^index\s*directiveId\s*eq" ${SLAPD_CONF_BKP} 2>/dev/null || true)
      INDEX_BKP_COUNT_2=$(grep -c "^index\s*modifyTimestamp\s*eq" ${SLAPD_CONF_BKP} 2>/dev/null || true)
      if [ ${INDEX_BKP_COUNT} -eq 0 || ${INDEX_BKP_COUNT_2} -eq 0 ]; then
        NEED_REINDEX=true
      fi
    fi
  fi

  if ${NEED_REINDEX}; then
    # stop slapd, slapindex -q, start slapd. 
    systemctl stop rudder-slapd
    echo -n "Reindexing LDAP directory - this may take a few minutes..."
    su - rudder-slapd -s /bin/sh -c "/opt/rudder/sbin/slapindex"
    echo " Done"
    systemctl start rudder-slapd
  fi
} 




################################################################################
# Check and upgrade plugins
################################################################################
upgrade_plugins() {
  STEP="Upgrade all plugins"
  if rudder package check-connection >/dev/null 2>&1 ; then
    rudder package update  || true
    rudder package upgrade-all  || true
  else
    echo "Plugins were not updated, because connection test is failing, please upgrade your plugins manually"  
  fi
  rudder package check-compatibility || true
}

################################################################################
# Migration function calls
################################################################################

update_credentials
upgrade_slapdconf
set_git_branch
upgrade_apache_conf
upgrade_system_techniques
upgrade_techniques
upgrade_ncf
upgrade_masterfiles
upgrade_rudder_web_properties

# Check the daemon status before upgrading
RUDDER_DB_INITIALIZED=true
echo -n "INFO: Checking PostgreSQL service status..."

# All versions: If the server should be on this machine, make sure rudder-reports is available - if not the DB will simply not be initialised yet
if [ "${SQL_SERVER}" = "localhost" ] && [ ! -f ${RUDDER_OPT}/etc/server-roles.d/rudder-reports ]; then
  # Server is localhost but rudder-reports is not installed - this is probably a half-installed server, warn and give up
  RUDDER_DB_INITIALIZED=false
  echo " Not installed"
  echo "INFO: No local database found. During initial install, this is expected."
fi

# All version: upgrade database if it is not empty
if ${RUDDER_DB_INITIALIZED}; then
  if retry_wrapper "${PSQL} -t -d ${SQL_DATABASE} -c 'select count(ruleid) from rules' >/dev/null 2>&1" "PostgreSQL";
  then
    echo " OK"
    upgrade_database
  else
    INCOMPLETE_MIGRATION=true
  fi
fi

# Check the daemon status before upgrading
RUDDER_LDAP_INITIALIZED=true
echo -n "INFO: Checking LDAP service status..."

# All version: upgrade ldap if it is not empty
if ${RUDDER_LDAP_INITIALIZED}; then
  # Using retry_wrapper is necessary because during an upgrade, slapd may be restarted, so we need to wait
  # for it to be available again
  if retry_wrapper "${LDAPSEARCH} -b '' -s base + >/dev/null 2>&1" "LDAP";
  then
    echo " OK"
    upgrade_inventory
  else
    INCOMPLETE_MIGRATION=true
  fi
fi

upgrade_plugins

# 6.1.5: Flag file /opt/rudder/etc/policy-update-running is not used anymore
rm -f /opt/rudder/etc/policy-update-running || true

################################################################################
# End
################################################################################

# For every upgrade, we force the root server to run a new inventory on the next CFEngine run
touch /opt/rudder/etc/force_inventory

echo ""
if ${INCOMPLETE_MIGRATION}
then
  echo "ERROR: The migration has failed in some steps. Check previous error messages."
  echo "Please restart the failed service(s), and start the migration script again."
  echo "(on a single Rudder server, try service rudder restart)"
  echo "Once it is working, run:"
  echo "# /opt/rudder/bin/rudder-upgrade"
else
  echo "INFO: The migration has completed successfully."
fi