#!/bin/bash
#purpose: backup mediawiki installation and update to latest release
#author(s): Guenther Schmitz <https://www.gpunktschmitz.com>
#license: CC0 1.0 Universal <https://creativecommons.org/publicdomain/zero/1.0/>

## Future:
# * Full dry run procedure
# * INPUT ARGUMENTS may be provided in command line or fall back to +INPUT ARGUMENTS section below
## Changes:
# 2017-11-21, by Pitchke
# * General functions allows modular process depending the context and ease debugging
# * The upgrade approch is NOT to overwrite previous mediawiki folder, but use a clean extraction then adding the required previous files (LocalSetting.php,https://releases.wikimedia.org/mediawiki/1.29/ images, extensions, etc.)
# * DO_NOT_EXIT_ON_NO_NEWER_VERSION, when true allows to reinstall the same version from other distribution (i.e. https://releases.wikimedia.org/mediawiki/1.29/ instead of  https://github.com/wikimedia/mediawiki/archive/${LATEST_RELEASE}.tar.gz)

# +PROCESS FLAGS (useful while debugging)
BACKUP_MYSQL_DB=false
BACKUP_WIKI_FILES=false
DO_NOT_EXIT_ON_NO_NEWER_VERSION=true
#TODO SKIP_RC_RELEASE=true
# -PROCESS FLAGS

# +INPUT ARGUMENTS
MEDIAWIKIDIR=/home/letitbefhm/www/home
BACKUPDIR=/home/letitbefhm/_MyBackup/MEDIAWIKIUPDATER
# +INPUT ARGUMENTS

# +VARIABLES
# Yes, /tmp is available on OVH( 2017-11-17)
TMPDIR=/tmp/MEDIAWIKIUPDATERTMP
#TODO CHOWNUSER=wwwrun
#TODO CHOWNGROUP=wwwrun
AWKEXECUTABLE=$(which awk)
SEDEXECUTABLE=$(which sed)
TAREXECUTABLE=$(which tar)
GREPEXECUTABLE=$(which grep)
WGETEXECUTABLE=$(which wget)
UNZIPEXECUTABLE=$(which unzip)
GUNZIPEXECUTABLE=$(which gunzip)
MYSQLDUMPEXECUTABLE=$(which mysqldump)
GITEXECUTABLE=$(which git)
MKDIREXECUTABLE=$(which mkdir)
PHPEXECUTABLE=$(which php)
XARGSEXECUTABLE=$(which xargs)
COMMEXECUTABLE=$(which comm)
COMPOSEREXECUTABLE=$(which composer)
# pushd seems an "internal" command, not an executable (on OVH 2017-11-17)
PUSHDEXECUTABLE=pushd
# popd seems an "internal" command, not an executable (on OVH 2017-11-17)
POPDEXECUTABLE=popd
HEADEXECUTABLE=$(which head)
# -VARIABLES

# +FUNCTIONS
#version compare function from the interweb (slightly modified to make the call more readable 'testvercomp "1.23.1" "<" "1.28.1"' but i really don't know how/why it works how it is now)
#https://stackoverflow.com/questions/4023830/how-compare-two-strings-in-dot-separated-version-format-in-bash?answertab=votes#tab-top
vercomp () {
    if [[ $1 == $2 ]]
    then
        return 0
    fi
    local IFS=.
    local i ver1=($1) ver2=($2)
    # fill empty fields in ver1 with zeros
    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
    do
        ver1[i]=0
    done
    for ((i=0; i<${#ver1[@]}; i++))
    do
        if [[ -z ${ver2[i]} ]]
        then
            # fill empty fields in ver2 with zeros
            ver2[i]=0
        fi
        if ((10#${ver1[i]} > 10#${ver2[i]}))
        then
            return 1
        fi
        if ((10#${ver1[i]} < 10#${ver2[i]}))
        then
            return 2
        fi
    done
    return 0
}

testvercomp () {
    vercomp $1 $3
    case $? in
        0) op='=';;
        1) op='>';;
        2) op='<';;
    esac
    if [[ $op != $2 ]]
    then
        return 0
    else
        return 1
    fi
}

backupmysqldb () {
	#backup mediawiki database
    echo "trying to backup database ..."
	if ${BACKUP_MYSQL_DB}; then
		DB_TYPE=$(${GREPEXECUTABLE} -n "wgDBtype =" ${MEDIAWIKIDIR}/LocalSettings.php | $AWKEXECUTABLE -F '"' '{print $2}')
		echo "database type: ${DB_TYPE}"

		if [ "$DB_TYPE" = "mysql" ]; then
			local DB_SERVER=$(${GREPEXECUTABLE} "wgDBserver =" ${MEDIAWIKIDIR}/LocalSettings.php)
			local DB_SERVER=${DB_SERVER:15:-2}
			local DB_NAME=$(${GREPEXECUTABLE} "wgDBname =" ${MEDIAWIKIDIR}/LocalSettings.php)
			local DB_NAME=${DB_NAME:13:-2}
			local DB_USER=$(${GREPEXECUTABLE} "wgDBuser =" ${MEDIAWIKIDIR}/LocalSettings.php)
			local DB_USER=${DB_USER:13:-2}
			local DB_PW=$(${GREPEXECUTABLE} "wgDBpassword =" ${MEDIAWIKIDIR}/LocalSettings.php)
			local DB_PW=${DB_PW:17:-2}

			echo "backing up database to '${BACKUPDIR}/${DB_NAME}.sql'"
			${MYSQLDUMPEXECUTABLE} -h ${DB_SERVER} -u ${DB_USER} -p${DB_PW} ${DB_NAME} > ${BACKUPDIRTIMESTAMP}/${DB_NAME}.sql
		else
			echo "database type is not 'mysql' (set BACKUP_MYSQL_DB to 'false' to continue without a backup created by this script) -> exiting"
			exit 1
		fi
	fi
}
# -FUNCTIONS


# + PROCESS SEQUENCE FUNCTIONS

CheckForLocalSettingsPhp () {
	echo "Checking for ${MEDIAWIKIDIR}/LocalSettings.php ..."
	if [ ! -f ${MEDIAWIKIDIR}/LocalSettings.php ]; then
		echo "file 'LocalSettings.php' not found in directory '${MEDIAWIKIDIR}'!"
		exit 1
	fi
	echo "-----"
}

GetMediaWikiVersion () {
	echo "getting mediawiki version from ${MEDIAWIKIDIR} ..."
	#get mediawiki version from MEDIAWIKIDIR
	INSTALLED_VERSION=$(${GREPEXECUTABLE} -n "wgVersion =" ${MEDIAWIKIDIR}/includes/DefaultSettings.php | ${AWKEXECUTABLE} -F "'" '{print $2}')
	echo "currently installed version: ${INSTALLED_VERSION}"

	LATEST_RELEASE=${INSTALLED_VERSION}

	#get releases from https://github.com/wikimedia/mediawiki/releases.atom
	LATEST_RELEASES=$(${WGETEXECUTABLE} -q -O- "https://github.com/wikimedia/mediawiki/releases.atom" | ${GREPEXECUTABLE} -o -P '<title>[^"]*' | ${SEDEXECUTABLE} "s/<title>//g" | ${SEDEXECUTABLE} "s/<\/title>//g")
	for RELEASE in ${LATEST_RELEASES}; do
		if [[ "${RELEASE}" =~ [0-9] ]]; then
		#skip if release candidate
			if ! ${GREPEXECUTABLE} "\-rc\." <<< ${RELEASE} &>/dev/null; then
				if testvercomp ${RELEASE} "<" ${LATEST_RELEASE}; then
					LATEST_RELEASE=${RELEASE}
				fi
			fi
		fi
	done

	echo "latest release found: ${LATEST_RELEASE}"
	
	if ! ${DO_NOT_EXIT_ON_NO_NEWER_VERSION}; then
		#check if latest version is newer
		if testvercomp ${LATEST_RELEASE} ">" ${INSTALLED_VERSION}; then
			echo "no newer version found on the interweb -> exiting"
			exit 1
		fi
	fi
	echo "-----"
}

DirectoriesCreation () {
	echo "Creating new unique directories (Temp, Backup) ..."

	#if tmp directory exists delete it/append "_{int}"
	if [ -d ${TMPDIR} ]; then
		COUNTER=1
		TMPBASEDIR=${TMPDIR}
		while [ -d ${TMPDIR} ]; do
			TMPDIR="${TMPBASEDIR}_${COUNTER}"
			COUNTER=$[COUNTER + 1]
		done
	fi

	#create tmp directory if not exists
	if [ ! -d ${TMPDIR} ]; then
		${MKDIREXECUTABLE} ${TMPDIR}
		echo "temp directory '${TMPDIR}' created"
	fi

	#create backup directory if not exists
	if [ ! -d ${BACKUPDIR} ]; then
		${MKDIREXECUTABLE} ${BACKUPDIR}
		echo "backup directory '${BACKUPDIR}' created"
	fi

	if ${BACKUP_MYSQL_DB} ]] || ${BACKUP_WIKI_FILES}; then
		#create backup timestamp directory
		TIMESTAMP=$(date +%Y%m%d_%H%M%S)
		BACKUPDIRTIMESTAMP="${BACKUPDIR}/${TIMESTAMP}"
		echo ${BACKUPDIRTIMESTAMP}
		if [ ! -d ${BACKUPDIRTIMESTAMP} ]; then
			${MKDIREXECUTABLE} ${BACKUPDIRTIMESTAMP}
			echo "backup directory '${BACKUPDIRTIMESTAMP}' created"
		fi
	fi
	echo "-----"
}

BackupMediawikiFiles () {
	echo "Backup MediaWiki files ..."
	#backup mediawiki files
	if ${BACKUP_WIKI_FILES}; then
		#copy $MEDIAWIKIDIR to $BACKUPDIR
		echo "copying '${MEDIAWIKIDIR}' to '${BACKUPDIRTIMESTAMP}/${INSTALLED_VERSION}'"
		${MKDIREXECUTABLE} ${BACKUPDIRTIMESTAMP}/${INSTALLED_VERSION}
		cp -r ${MEDIAWIKIDIR}/* ${BACKUPDIRTIMESTAMP}/${INSTALLED_VERSION}/
	fi
	echo "-----"
}		

DownloadNewMediaWiki-Original () {
	## This may not be the best source/download version to use as it does not include Skins and some Extensions for example which I had to copy back manually (found because update.php was doing nothing)
	## Prefered bundle source: https://releases.wikimedia.org/mediawiki/1.29/
	
	echo "Downloading and extracting new MediaWiki files ..."
	if [[ ${UNZIPEXECUTABLE} ]]; then
		#download new mediawiki version
		echo "downloading version '${LATEST_RELEASE}' (zip)"
		${WGETEXECUTABLE} https://github.com/wikimedia/mediawiki/archive/${LATEST_RELEASE}.zip -O ${TMPDIR}/${LATEST_RELEASE}.zip -o /dev/null

		if ${UNZIPEXECUTABLE} -l ${TMPDIR}/${LATEST_RELEASE}.zip &> /dev/null; then
			#extract downloaded release
			echo "extracting ${LATEST_RELEASE}"
			${UNZIPEXECUTABLE} -oq ${TMPDIR}/${LATEST_RELEASE}.zip -d ${TMPDIR}
		else
			echo "download (${TMPDIR}/${LATEST_RELEASE}.zip) seems not to be a valid zip file"
		fi
	elif [[ ${TAREXECUTABLE} ]] && [[ ${GUNZIPEXECUTABLE} ]]; then
		#download new mediawiki version
		echo "downloading version '${LATEST_RELEASE}' (tar.gz)"
		${WGETEXECUTABLE} https://github.com/wikimedia/mediawiki/archive/${LATEST_RELEASE}.tar.gz -O ${TMPDIR}/${LATEST_RELEASE}.tar.gz -o /dev/null
		
		#extract downloaded release
		echo "extracting $LATEST_RELEASE"                                                                  
		${PUSHDEXECUTABLE} ${TMPDIR}
		if $(${TAREXECUTABLE} -tzf ${LATEST_RELEASE}.tar.gz &>/dev/null); then                                                                                      
			${TAREXECUTABLE} -xzf ${LATEST_RELEASE}.tar                                                                                        
			cd $(${POPDEXECUTABLE})
		else
			cd $(%{POPDEXECUTABLE})
			echo "download (${TMPDIR}/${LATEST_RELEASE}.tar.gz) seems not to be a valid tar.gz file"
		fi
	else
		echo "no tool (unzip or tar) found .. cannot extract new release."
		exit 1
	fi
	echo "-----"
}

DownloadNewMediaWiki () {
	## This may not be the best source/download version to use as it does not include Skins and some Extensions for example which I had to copy back manually (found because update.php was doing nothing)
	## Prefered bundle source: https://releases.wikimedia.org/mediawiki/1.29/
	
	echo "Downloading and extracting new MediaWiki files ..."
	echo "\${LATEST_RELEASE}=${LATEST_RELEASE}"
	echo "\${TMPDIR}=${TMPDIR}"	
	echo "\${TAREXECUTABLE}=${TAREXECUTABLE}"
	echo "\${GUNZIPEXECUTABLE}=${GUNZIPEXECUTABLE}"
	
	if  [[ ${TAREXECUTABLE} ]] && [[ ${GUNZIPEXECUTABLE} ]]; then
		#download new mediawiki version
		echo "downloading version '${LATEST_RELEASE}' (tar.gz)"
		# was i.e.     https://github.com/wikimedia/mediawiki/archive/1.29.2.tar.gz
		# what I want: https://releases.wikimedia.org/mediawiki/1.29/mediawiki-1.29.2.tar.gz
		${WGETEXECUTABLE} https://releases.wikimedia.org/mediawiki/1.29/mediawiki-1.29.2.tar.gz -O ${TMPDIR}/${LATEST_RELEASE}.tar.gz -o /dev/null
		#${WGETEXECUTABLE} https://github.com/wikimedia/mediawiki/archive/${LATEST_RELEASE}.tar.gz -O ${TMPDIR}/${LATEST_RELEASE}.tar.gz -o /dev/null
		
		#extract downloaded release
		echo "extracting $LATEST_RELEASE"                                                                  
		${PUSHDEXECUTABLE} ${TMPDIR}
		if $(${TAREXECUTABLE} -tzf ${LATEST_RELEASE}.tar.gz &>/dev/null); then                 
			echo "extraction was working"
			${TAREXECUTABLE} -xzf ${LATEST_RELEASE}.tar.gz
			echo "extracted content"
			${POPDEXECUTABLE}
		else
			${POPDEXECUTABLE}
			echo "download (${TMPDIR}/${LATEST_RELEASE}.tar.gz) seems not to be a valid tar.gz file"
		fi
	else
		echo "no tool (unzip or tar) found .. cannot extract new release."
		exit 1
	fi
	echo "-----"
}

MoveOldFiles () {
	echo "Moving old MediaWiki files (only some will be restored later) ..."
	##  "It's recommended that you unpack the new files into a new directory, and then apply customizations to the new directory (restoring LocalSettings.php, images folder, extensions, and other customizations like custom skins)" https://www.mediawiki.org/wiki/Manual:Upgrading#Unpack_the_new_files

	# Moving all files from MEDIAWIKIDIR to MEDIAWIKIDIR.NotRestored,
	# Then a clean extraction to MEDIAWIKIDIR is performed as recommended,
	# and the required files (LocalSetings.php, composer.local.json, images, additional extensions etc.) will be MOVED from MEDIAWIKIDIR.NotRestored to MEDIAWIKIDIR. This allows to further investigate on missing files (from MEDIAWIKIDIR.NotRestored) as required.
	# Once really completed and tested, MEDIAWIKIDIR.NotRestored will be removed (manually)

	# a file move is prefered just to avoid impacting permissions on moving/creating directories.
	if [ ! -d "${MEDIAWIKIDIR}.NotRestored" ]; then
		${MKDIREXECUTABLE} "${MEDIAWIKIDIR}.NotRestored"
		echo "backup directory ${MEDIAWIKIDIR}.NotRestored created"
	else
		echo "Aborted: backup directory ${MEDIAWIKIDIR}.NotRestored exists."
		exit 1
	fi

	echo "Moving old files to ${MEDIAWIKIDIR}.NotRestored ..."
	mv ${MEDIAWIKIDIR}/* "${MEDIAWIKIDIR}.NotRestored/"
	echo "-----"
}

UpdatingInstallationWithNewFiles () {
	echo "Installing new MediaWiki files ..."
	#get extracted directory
	NEW_RELEASE_DIRECTORY=$(ls -d ${TMPDIR}/*/)
	echo "\${NEW_RELEASE_DIRECTORY}=${NEW_RELEASE_DIRECTORY}"
	if [[ ${NEW_RELEASE_DIRECTORY} ]]; then
		echo "new release extracted to '${NEW_RELEASE_DIRECTORY}'"

		#overwrite current installation
		echo "updating installation with new files"
		cp -r ${NEW_RELEASE_DIRECTORY}/* ${MEDIAWIKIDIR}
	fi
	echo "-----"
}

UpdatingDependencies () {
	# I prefer using Composer (see composer-update () below)
	
	# Not working,
	## running: git clone https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git
	## gives: fatal: unable to access 'https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git/': Failed to connect to gerrit.wikimedia.org port 443: Connection refused

	echo "Updating dependencies (./vendor) ..."
	#updating dependencies
	echo "deleting path ${MEDIAWIKIDIR}/vendor"
	rm -rf ${MEDIAWIKIDIR}/vendor
	echo "cloning repo 'https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git'"
	if [[ ${GITEXECUTABLE} ]]; then
		${GITEXECUTABLE} clone https://gerrit.wikimedia.org/r/p/mediawiki/vendor.git ${MEDIAWIKIDIR}/vendor
	else
		echo "git executable not found - the vendors could not be downloaded"
	fi
	echo "-----"
}

RestoreOldRequiredFilesToNewWiki () {
	##  "It's recommended that you unpack the new files into a new directory, and then apply customizations to the new directory (restoring LocalSettings.php, images folder, extensions, and other customizations like custom skins)" https://www.mediawiki.org/wiki/Manual:Upgrading#Unpack_the_new_files

	echo "Restore some old required files in new MediaWiki ..."
	
	#prerequisites	
	if [[ ${MEDIAWIKIDIR} ]]; then
		echo "New mediawiki dir is  '${MEDIAWIKIDIR}' (\${MEDIAWIKIDIR})"
		# test if exist:
	else
		echo "New mediawiki dir is undefined (\${MEDIAWIKIDIR})"
		exit 1
	fi
	# test if exist:
	# "${MEDIAWIKIDIR}.NotRestored" assumed

	# Restore LocalSettings.php, composer.local.json, composer.lock
	# ToDo: test if not exist 
	mv "${MEDIAWIKIDIR}.NotRestored/LocalSettings.php" "${MEDIAWIKIDIR}/"
	mv "${MEDIAWIKIDIR}.NotRestored/composer.local.json" "${MEDIAWIKIDIR}/"
	# Unclear to me if composer.lock is to be preserved or recreated (by 'composer update')
	mv "${MEDIAWIKIDIR}.NotRestored/composer.lock" "${MEDIAWIKIDIR}/"
	
	# Restore misc files I want to keep
	for filename in ${MEDIAWIKIDIR}.NotRestored/*.ico ; do
		echo "${filename}"
		mv "${filename}" "${MEDIAWIKIDIR}/"
	done

	#TODO chown
	# The images, and change the ownership and permissions. find ./images -type d -exec chmod 755 {} \; and chgrp -R apache images (e.g. if your web user is apache).
	echo "Restoring images ..."
	##### only once !!!! rm -r "${MEDIAWIKIDIR}/images"
	rm -r "${MEDIAWIKIDIR}/images"
	mv -v "${MEDIAWIKIDIR}.NotRestored/images/" "${MEDIAWIKIDIR}/"

	# Some extensions in the extensions directory. You should always get updated extensions, old extensions aren't guaranteed to work with a newer version of MediaWiki.
	echo "Restoring extensions ..."		
	list=$(${COMMEXECUTABLE} -23 <(ls ${MEDIAWIKIDIR}.NotRestored/extensions|sort) <(ls ${MEDIAWIKIDIR}/extensions|sort))	
	OLDIFS=${IFS}
	IFS='
	'
	for pathname in ${list} ; do
		echo "about to move: mv " "${MEDIAWIKIDIR}.NotRestored/extensions/${pathname}" "${MEDIAWIKIDIR}/extensions"
		mv "${MEDIAWIKIDIR}.NotRestored/extensions/${pathname}" "${MEDIAWIKIDIR}/extensions"
	done
	IFS=${OLDIFS}
	
	# ToDo/NiceTo: If possible, a list of files still present in MEDIAWIKIDIR.NotRestored and not in the new MEDIAWIKIDIR should be reported.

	# In case of specific customisation i.e. changes mades on the past MediaWiki specific php code or common css etc. These files/changes could be mentioned here for reference and suggest their edition from the old file(s) to the new one(s).

	echo "-----"
}

composer-update () {
	# Automatically download/update dependencies as defined for MediaWiki (in composer.json)
	# Also automatically download/update dependencies as defined in your composer.local.json (if defined) for i.e. additional extensions.
	
	# See also UpdatingDependencies () as an alternative to composer-update ()
	
	echo "Running composer update (from ${MEDIAWIKIDIR}) ..."
	${PUSHDEXECUTABLE} ${MEDIAWIKIDIR}
	#${COMPOSEREXECUTABLE} update --dry-run
	${COMPOSEREXECUTABLE} update
	${POPDEXECUTABLE}
	echo "-----"
}

update.php () {
	# This is the main MediaWiki database update/upgrade process
	echo "Running update.php ..."
	${PUSHDEXECUTABLE} ${MEDIAWIKIDIR}
	${PHPEXECUTABLE} ${MEDIAWIKIDIR}/maintenance/update.php
	#${PHPEXECUTABLE} ${MEDIAWIKIDIR}/maintenance/update.php --skip-external-dependencies
	${POPDEXECUTABLE}
	echo "-----"
}

SMW_refreshData () {
	# upgrading SMW requires running SMW_refreshData.php
	echo "Running SMW_refreshData.php ..."
	${PUSHDEXECUTABLE} ${MEDIAWIKIDIR}
	${PHPEXECUTABLE} ${MEDIAWIKIDIR}/extensions/SemanticMediaWiki/maintenance/SMW_refreshData.php
	${POPDEXECUTABLE}
	echo "-----"
}

runJobs.php () {
	# runJobs.php is also recommended prior to initiate an upgrade to ensure a stable/integrity state
	echo "Running runJobs.php ..."
	${PUSHDEXECUTABLE} ${MEDIAWIKIDIR}
	${PHPEXECUTABLE} ${MEDIAWIKIDIR}/maintenance/runJobs.php
	${POPDEXECUTABLE}
	echo "-----"
}
rebuildtextindex.php () {
	# Run rebuildtextindex.php if search does not find (some) pages/content
	echo "Running rebuildtextindex.php ..."
	${PUSHDEXECUTABLE} ${MEDIAWIKIDIR}
	${PHPEXECUTABLE} ${MEDIAWIKIDIR}/maintenance/rebuildtextindex.php
	${POPDEXECUTABLE}
	echo "-----"
}

# - PROCESS SEQUENCE FUNCTIONS


# +PROCESS
# See https://www.mediawiki.org/wiki/Manual:Upgrading as the reference manual for upgrading,
# and adapt below sequence depending your upgrade requirements.
# See comments inside each procedure for more details

#runJobs.php
#CheckForLocalSettingsPhp
GetMediaWikiVersion
DirectoriesCreation
#backupmysqldb
#BackupMediawikiFiles
DownloadNewMediaWiki
#MoveOldFiles
#UpdatingInstallationWithNewFiles
#UpdatingDependencies
#RestoreOldRequiredFilesToNewWiki

# Consider to iterate using next functions as required by:
# (1) Testing if the site is still running
# (2) Test your extensions and any expected behaviour in your site
# (3) run Composer to update some packages, then
# (4) possibly run Composer to upgrade package(s)
# (5) (if using and upgrading SMW) run SMW refreshData.php
# (6) Run rebuildtextindex.php if search does not find (some) pages/content

#composer-update
#update.php
#SMW_refreshData
#runJobs.php
#rebuildtextindex.php

exit 1

#remove old backup directory
echo "removing old backups (keeping last 3)"
ls -t -d ${BACKUPDIR}/*/  | ${GREPEXECUTABLE} -v "$(ls -t ${BACKUPDIR}/ | ${HEADEXECUTABLE} -3)" | ${XARGSEXECUTABLE} rm -r

#remove tmp directory
echo "removing tmp directory"
rm -r ${TMPDIR}

# -PROCESS