diff --git a/usr/share/rear/backup/DUPLICITY/default/50_make_duplicity_backup.sh b/usr/share/rear/backup/DUPLICITY/default/50_make_duplicity_backup.sh new file mode 100644 index 0000000000..4c41032630 --- /dev/null +++ b/usr/share/rear/backup/DUPLICITY/default/50_make_duplicity_backup.sh @@ -0,0 +1,120 @@ +# This file is part of Relax and Recover, licensed under the GNU General +# Public License. Refer to the included LICENSE for full text of license. + +if [ "$BACKUP_PROG" = "duply" ] && has_binary duply; then + # we found the duply program; check if a profile was defined + [[ -z "$DUPLY_PROFILE" ]] && return + + # a real profile was detected - check if we can talk to the remote site + duply "$DUPLY_PROFILE" backup >&2 # output is going to logfile + StopIfError "Duply profile $DUPLY_PROFILE backup returned errors - see $LOGFILE" + + LogPrint "The last full backup taken with duply/duplicity was:" + LogPrint "$( tail -10 $LOGFILE | grep Full )" +fi + + +if [ "$BACKUP_PROG" = "duplicity" ]; then + + # make backup using the DUPLICITY method with duplicity + # by falk hoeppner + + LogPrint "Creating $BACKUP_PROG archives on '$BACKUP_DUPLICITY_URL'" + + # using some new parameters from config.local + # + # DUPLICITY_USER + # DUPLICITY_HOST + # DUPLICITY_PROTO + # DUPLICITY_PATH + # BACKUP_DUPLICITY_URL + + if [ -n $DUPLICITY_USER -a -n $DUPLICITY_HOST -a -n $DUPLICITY_PROTO -a -n $DUPLICITY_PATH ] + then + BKP_URL="$BACKUP_DUPLICITY_URL" + # ToDo: do some more plausibility checks !? + else + Error "Parameters for BACKUP_DUPLICITY_URL not set correctly, please look at config.local template" + fi + + # todo: check parameters + DUP_OPTIONS="$BACKUP_DUPLICITY_OPTIONS" + # + GPG_OPT="${BACKUP_DUPLICITY_GPG_OPTIONS}" + GPG_KEY="$BACKUP_DUPLICITY_GPG_ENC_KEY" + PASSPHRASE="$BACKUP_DUPLICITY_GPG_ENC_PASSPHRASE" + + echo "GPG_OPT = $GPG_OPT" + + # EXCLUDES="${TMP_DIR}/backup_exclude.lst" + + # NMBRS=${#BACKUP_DUPLICITY_EXCLUDE[$@]} + # echo NMBRS = $NMBRS + + # for i in $(seq 0 $(($NMBRS - 1)) ) + # do + # LogPrint "Exclude No $i = ${BACKUP_DUPLICITY_EXCLUDE[$i]}" + # echo "${BACKUP_DUPLICITY_EXCLUDE[$i]}" >> "${EXCLUDES}" + # done + + # runs without external file, but all the * in the excludelist + # will expanded :-( + # + for EXDIR in ${BACKUP_DUPLICITY_EXCLUDE[@]} + do + EXCLUDES="$EXCLUDES --exclude $EXDIR" + done + + echo EXCLUDES = $EXCLUDES + + # Setting the pass phrase to encrypt the backup files + export PASSPHRASE + + # check and create directory at backup-server + # if the target-directory dont exist, duplicity will fail + # Note: this is only working if we use duplicity with ssh/rsync and the + # given user is allowed to create directories/files this way !! + # maybe better done in an if or case statment + # + LogPrint "Checking backup-path at server ..." + ssh ${DUPLICITY_USER}@${DUPLICITY_HOST} "test -d ${DUPLICITY_PATH}/${HOSTNAME} || mkdir -p ${DUPLICITY_PATH}/${HOSTNAME}" + + # first remove everything older than $BACKUP_DUPLICITY_MAX_TIME + if [ -z $BACKUP_DUPLICITY_MAX_TIME ] + then + BACKUP_DUPLICITY_MAX_TIME=2M # Default: alte Backups nach 2 Monaten löschen + fi + LogPrint "Removing the old stuff from server with CMD: + $DUPLICITY_PROG remove-older-than $BACKUP_DUPLICITY_MAX_TIME -v5 $BKP_URL/$HOSTNAME" + $DUPLICITY_PROG remove-older-than $BACKUP_DUPLICITY_MAX_TIME -v5 $BKP_URL/$HOSTNAME >> ${TMP_DIR}/${BACKUP_PROG_ARCHIVE}.log + + # do the backup + LogPrint "Running CMD: $DUPLICITY_PROG -v5 $DUP_OPTIONS --encrypt-key $GPG_KEY $GPG_OPT \ + --exclude '/proc/*' --exclude '/sys/*' --exclude '/var/lib/ntp/proc/*' $EXCLUDES \ + / $BKP_URL/$HOSTNAME >> ${TMP_DIR}/${BACKUP_PROG_ARCHIVE}.log 2>&1" + $DUPLICITY_PROG -v5 $DUP_OPTIONS --encrypt-key $GPG_KEY $GPG_OPT \ + --exclude '/proc/**' --exclude '/sys/**' --exclude '/var/lib/ntp/proc/**' $EXCLUDES \ + / $BKP_URL/$HOSTNAME >> ${TMP_DIR}/${BACKUP_PROG_ARCHIVE}.log 2>&1 + + RC_DUP=$? + + sleep 1 + LOGAUSZUG=$(tail -n 18 ${TMP_DIR}/${BACKUP_PROG_ARCHIVE}.log) + LogPrint "${LOGAUSZUG}" + + # everyone should see this warning, even if not verbose + if [ $RC_DUP -gt 0 ] + then + VERBOSE=1 + LogPrint "WARNING ! + There was an error during archive creation. + Please check the archive and see '$LOGFILE' for more information. + + Since errors are oftenly related to files that cannot be saved by + $BACKUP_PROG, we will continue the $WORKFLOW process. However, you MUST + verify the backup yourself before trusting it ! + " + LogPrint "$LOGAUSZUG" + fi +fi + diff --git a/usr/share/rear/backup/DUPLICITY/default/50_make_duply_backup.sh b/usr/share/rear/backup/DUPLICITY/default/50_make_duply_backup.sh index 28e6bb1bb3..1fd153de67 100644 --- a/usr/share/rear/backup/DUPLICITY/default/50_make_duply_backup.sh +++ b/usr/share/rear/backup/DUPLICITY/default/50_make_duply_backup.sh @@ -1,7 +1,7 @@ # This file is part of Relax and Recover, licensed under the GNU General # Public License. Refer to the included LICENSE for full text of license. -if has_binary duply; then +if [ "$BACKUP_PROG" = "duply" ] && has_binary duply; then # we found the duply program; check if a profile was defined [[ -z "$DUPLY_PROFILE" ]] && return diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 4aee18b8c4..8640313328 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -476,8 +476,59 @@ BEXTRACT_DEVICE= # to define your settings - use it as "duply status" to see it in action) # By using DUPLY_PROFILE we will try an automatic restore, if duplicity directly is used # then you better add some restore script in the COPY_AS_IS array +# +# BACKUP_PROG="duply" DUPLY_PROFILE="" +####################################################################### +# Extention for DUPLICITY +# +# BACKUP=DUPLICITY +# BACKUP_PROG="duplicity" +# +# DUPLICITY_PROG="/usr/bin/duplicity" +# +# the ssh/rsync user at the backup server that is allowed to read/write the Backup +# for this host +# +# DUPLICITY_USER="" +# +# the backup server to write/read the backup: +# +# DUPLICITY_HOST="" +# +# the protocol duplicity should use for backup/restore (must be rsync for this case) +# duplicity supports much more (ssh, scp, ftp, ..) but this isnt test in rear yet +# +# DUPLICITY_PROTO="rsync" +# +# the path at the remote host, that contain the host-dirs with the backups +# +# DUPLICITY_PATH="" +# +# combining the upper parameters +# should result e.g. in BACKUP_DUPLICITY_URL="rsync://rear-user@192.168.99.10//backup/rear" +# +# BACKUP_DUPLICITY_URL="${DUPLICITY_PROTO}://${DUPLICITY_USER}@${DUPLICITY_HOST}/${DUPLICITY_PATH}" +# +# value for duplicity action 'remove-older-than' +# for format of time see the TIME FORMATS section at man duplicity +# default = 2M ^= 2 Month +# +# BACKUP_DUPLICITY_MAX_TIME="2M" + +# GPG-KEY for encrypt backup (e.g. for backup-User) +# BACKUP_DUPLICITY_GPG_OPTIONS="--gpg-options '--cipher-algo AES'" +# BACKUP_DUPLICITY_GPG_ENC_KEY="" +# BACKUP_DUPLICITY_GPG_ENC_PASSPHRASE="passphrase" + +# GPG-KEY for sign backup (e.g. root-User) +# BACKUP_DUPLICITY_GPG_SIGN_KEY="" + +# Some extra options for duplicity, see man duplicity. +# BACKUP_DUPLICITY_OPTIONS="--volsize 100" +# +####################################################################### ## # BACKUP=NETFS stuff diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh index f286a57dd2..fee8af19f4 100644 --- a/usr/share/rear/lib/global-functions.sh +++ b/usr/share/rear/lib/global-functions.sh @@ -20,9 +20,9 @@ function read_and_strip_file () { # extracts content from config files. In other words: strips the comments and new lines - if test -s "$1" ; then - sed -e '/^[[:space:]]/d;/^$/d;/^#/d' "$1" - fi + if test -s "$1" ; then + sed -e '/^[[:space:]]/d;/^$/d;/^#/d' "$1" + fi } function is_numeric () { @@ -41,8 +41,7 @@ function is_numeric () { url_scheme() { local url=$1 local scheme=${url%%://*} - # rsync scheme does not have to start with rsync:// it can also be scp style (old style) - # next line will be removed in future releases (TODO) + # rsync scheme does not have to start with rsync:// it can also be scp style echo $scheme | grep -q ":" && echo rsync || echo $scheme } @@ -138,9 +137,12 @@ mount_url() { (usb) mount_cmd="mount $v -o $options $(url_path $url) $mountpoint" ;; - (sshfs) - mount_cmd="sshfs $(url_host $url):$(url_path $url) $mountpoint -o $options" + (sshfs) + mount_cmd="sshfs $(url_host $url):$(url_path $url) $mountpoint -o $options" ;; + (davfs) + mount_cmd="mount $v -t davfs http://$(url_host $url)$(url_path $url) $mountpoint" + ;; (*) mount_cmd="mount $v -t $(url_scheme $url) -o $options $(url_host $url):$(url_path $url) $mountpoint" ;; @@ -169,9 +171,19 @@ umount_url() { return 0 fi ;; - (sshfs) - umount_cmd="fusermount -u $mountpoint" - ;; + (sshfs) + umount_cmd="fusermount -u $mountpoint" + ;; + (davfs) + umount_cmd="umount $mountpoint" + # Wait for 3 sek. then remove the cache-dir /var/cache/davfs + sleep 30 + # ToDo: put in here the cache-dir from /etc/davfs2/davfs.conf + # and delete only the just used cache + #rm -rf /var/cache/davfs2/** + rm -rf /var/cache/davfs2/*outputfs* + + ;; (var) local var=$(url_host $url) umount_cmd="${!var} $mountpoint" diff --git a/usr/share/rear/output/PXE/default/82_copy_to_net.sh b/usr/share/rear/output/PXE/default/82_copy_to_net.sh index dc71065cea..88a9eba925 100644 --- a/usr/share/rear/output/PXE/default/82_copy_to_net.sh +++ b/usr/share/rear/output/PXE/default/82_copy_to_net.sh @@ -10,7 +10,7 @@ local server=$(url_host $OUTPUT_URL) local path=$(url_path $OUTPUT_URL) case "$scheme" in - (nfs|cifs|usb|tape|file) + (nfs|cifs|usb|tape|file|davfs) # The ISO has already been transferred by NETFS. return 0 ;; diff --git a/usr/share/rear/output/RAMDISK/Linux-i386/90_copy_ramdisk.sh b/usr/share/rear/output/RAMDISK/Linux-i386/90_copy_ramdisk.sh index a74bc0d2d8..d30ae764d8 100644 --- a/usr/share/rear/output/RAMDISK/Linux-i386/90_copy_ramdisk.sh +++ b/usr/share/rear/output/RAMDISK/Linux-i386/90_copy_ramdisk.sh @@ -28,7 +28,7 @@ case "$scheme" in cp $v -pLf $KERNEL_FILE $path/kernel-$RAMDISK_SUFFIX >&2 cp $v -pLf $TMP_DIR/initrd.cgz $path/initramfs-$RAMDISK_SUFFIX.img >&2 ;; - (nfs|cifs|usb) + (nfs|cifs|usb|davfs) LogPrint "Transferring kernel and initramfs to $OUTPUT_URL" mkdir -p $v $BUILD_DIR/outputfs/$NETFS_PREFIX/ >&2 cp $v -pLf $KERNEL_FILE $BUILD_DIR/outputfs/$NETFS_PREFIX/kernel-$RAMDISK_SUFFIX >&2 diff --git a/usr/share/rear/output/default/95_copy_result_files.sh b/usr/share/rear/output/default/95_copy_result_files.sh index e7819f5447..40608f74fb 100644 --- a/usr/share/rear/output/default/95_copy_result_files.sh +++ b/usr/share/rear/output/default/95_copy_result_files.sh @@ -14,7 +14,7 @@ fi LogPrint "Copying resulting files to $scheme location" case "$scheme" in - (nfs|cifs|usb|file|sshfs) + (nfs|cifs|usb|file|sshfs|davfs) # if called as mkbackuponly then we just don't have any result files. if test "$RESULT_FILES" ; then Log "Copying files '${RESULT_FILES[@]}' to $scheme location" diff --git a/usr/share/rear/prep/DUPLICITY/default/05_prep_duply.sh b/usr/share/rear/prep/DUPLICITY/default/05_prep_duply.sh index 00300784cf..a8d0999687 100644 --- a/usr/share/rear/prep/DUPLICITY/default/05_prep_duply.sh +++ b/usr/share/rear/prep/DUPLICITY/default/05_prep_duply.sh @@ -1,7 +1,19 @@ # This file is part of Relax and Recover, licensed under the GNU General # Public License. Refer to the included LICENSE for full text of license. -REQUIRED_PROGS=( "${REQUIRED_PROGS[@]}" gpg duplicity python ) +# At SLES11 /usr/bin/python is a link to ./python2.6 +# and because copy failed with an error "cp: not writing through dangling symlink" +# we need to put in the link target ... + +PYTHON="$(which python)" + +if [ -h "$PYTHON" ]; then + PYTHON_BIN=$(basename $(readlink "$PYTHON")) +else + PYTHON_BIN="python" +fi + +REQUIRED_PROGS=( "${REQUIRED_PROGS[@]}" gpg duplicity "$PYTHON_BIN" ) # duply is a really good shell script wrapper for duplicity PROGS=( "${PROGS[@]}" duply ) @@ -16,6 +28,12 @@ COPY_AS_IS=( /usr/lib/pymodules /usr/lib/pyshared /usr/lib/python2.6 +/usr/lib64/python2.6 +/usr/lib64/python2.6/lib-dynload +/usr/lib64/python2.6/site-packages +/usr/lib64/python2.6/site-packages/gnupg.py +/usr/lib64/python2.6/site-packages/GnuPGInterface.py +/usr/lib64/python2.6/site-packages/duplicity /usr/lib/python2.7 /usr/lib64/python2.7 /usr/lib/python3.1 diff --git a/usr/share/rear/prep/DUPLICITY/default/20_find_duply_profile.sh b/usr/share/rear/prep/DUPLICITY/default/20_find_duply_profile.sh index 94bca7ccb6..4ccf82297c 100644 --- a/usr/share/rear/prep/DUPLICITY/default/20_find_duply_profile.sh +++ b/usr/share/rear/prep/DUPLICITY/default/20_find_duply_profile.sh @@ -8,7 +8,8 @@ # the profile is in fact a directory name containing the conf file and exclude file # we shall copy this variable, if defined, to our rescue.conf file -if has_binary duply; then +if [ "$BACKUP_PROG" = "duply" ] && has_binary duply; then + # we found the duply program; check if we can find a profile defined if [[ -z "$DUPLY_PROFILE" ]]; then # no profile pre-set; let's try to find one diff --git a/usr/share/rear/restore/DUPLICITY/default/10_restore_duply.sh b/usr/share/rear/restore/DUPLICITY/default/10_restore_duply.sh index 5062154da2..036def0891 100644 --- a/usr/share/rear/restore/DUPLICITY/default/10_restore_duply.sh +++ b/usr/share/rear/restore/DUPLICITY/default/10_restore_duply.sh @@ -1,31 +1,43 @@ # This file is part of Relax and Recover, licensed under the GNU General # Public License. Refer to the included LICENSE for full text of license. -[[ -z "$DUPLY_PROFILE" ]] && return -# we need to restore on a path that does not exist ;-/ -# that is why we add "restore" to /mnt/local -duply "$DUPLY_PROFILE" restore /mnt/local/restore -if (( $? > 1 )); then - LogPrintIfError "duply $DUPLY_PROFILE restore /mnt/local failed" - DUPLY_RESTORE_OK="n" -else - DUPLY_RESTORE_OK="y" +if [ "$BACKUP_PROG" = "duply" ] && has_binary duply; then - # we need to move up one dir (to get restore almost empty) - cd /mnt/local - - # file $VAR_DIR/recovery/mountpoint_device contains the mount points in / /boot etc order - # we need to reverse it - to avoid tac we use sed instead - for mntpt in $( awk '{print $1}' $VAR_DIR/recovery/mountpoint_device | sed -n '1!G;h;$p' ) - do - mv restore${mntpt}/* .${mntpt} >&2 # mv restore/boot/* ./boot - done + if [ -z "$DUPLY_PROFILE" ]; then + # Backup-Method is DUPLICITY with DUPLY Frontend but no Duply-Profile given + # so we have to do the restore manually + LogPrint "Restore with Duply failed, no DUPLY_PROFILE given." + return + else + + # we need to restore on a path that does not exist ;-/ + # that is why we add "restore" to /mnt/local + duply "$DUPLY_PROFILE" restore /mnt/local/restore + if (( $? > 1 )); then + LogPrintIfError "duply $DUPLY_PROFILE restore /mnt/local failed" + DUPLY_RESTORE_OK="n" + else + DUPLY_RESTORE_OK="y" + + # we need to move up one dir (to get restore almost empty) + cd /mnt/local + + # file $VAR_DIR/recovery/mountpoint_device contains the mount points in / /boot etc order + # we need to reverse it - to avoid tac we use sed instead + for mntpt in $( awk '{print $1}' $VAR_DIR/recovery/mountpoint_device | sed -n '1!G;h;$p' ) + do + mv restore${mntpt}/* .${mntpt} >&2 # mv restore/boot/* ./boot + done + + # double check on some important moint-points + [[ ! -d /mnt/local/mnt ]] && mkdir -m 755 /mnt/local/mnt + [[ ! -d /mnt/local/proc ]] && mkdir -m 555 /mnt/local/proc + [[ ! -d /mnt/local/tmp ]] && mkdir -m 4777 /mnt/local/tmp + + cd - >/dev/null + fi + fi +fi - # double check on some important moint-points - [[ ! -d /mnt/local/mnt ]] && mkdir -m 755 /mnt/local/mnt - [[ ! -d /mnt/local/proc ]] && mkdir -m 555 /mnt/local/proc - [[ ! -d /mnt/local/tmp ]] && mkdir -m 4777 /mnt/local/tmp - cd - >/dev/null -fi diff --git a/usr/share/rear/restore/DUPLICITY/default/20_prompt_user_to_start_restore.sh b/usr/share/rear/restore/DUPLICITY/default/20_prompt_user_to_start_restore.sh index c368dc181b..2d8a0116b6 100644 --- a/usr/share/rear/restore/DUPLICITY/default/20_prompt_user_to_start_restore.sh +++ b/usr/share/rear/restore/DUPLICITY/default/20_prompt_user_to_start_restore.sh @@ -4,10 +4,19 @@ # the user has to do the main part here :-) # # -[[ "$DUPLY_RESTORE_OK" = "y" ]] && return -LogPrint "Please restore your backup in the provided shell to /mnt/local and, when finished, type exit -in the shell to continue recovery. You can use duplicity / duply to restore your backup." +if [ "$BACKUP_PROG" = "duply" ]; then -export TMPDIR=/mnt/local -rear_shell "Did you restore the backup to /mnt/local ? Are you ready to continue recovery ?" + [[ "$DUPLY_RESTORE_OK" = "y" ]] && return + + # if restore should be done with duply, but it failed, give the user + # a chance to fix it manually + + LogPrint "Please restore your backup in the provided shell to /mnt/local and, + when finished, type exit in the shell to continue recovery. + You can use duplicity / duply to restore your backup." + + export TMPDIR=/mnt/local + rear_shell "Did you restore the backup to /mnt/local ? Are you ready to continue recovery ?" + +fi diff --git a/usr/share/rear/restore/DUPLICITY/default/40_restore_duplicity.sh b/usr/share/rear/restore/DUPLICITY/default/40_restore_duplicity.sh new file mode 100644 index 0000000000..7b8d598962 --- /dev/null +++ b/usr/share/rear/restore/DUPLICITY/default/40_restore_duplicity.sh @@ -0,0 +1,66 @@ +# This file is part of Relax and Recover, licensed under the GNU General +# Public License. Refer to the included LICENSE for full text of license. +# +# Restore from remote backup via DUPLICIY over rsync + +if [ "$BACKUP_PROG" = "duplicity" ]; then + + LogPrint "========================================================================" + LogPrint "Restoring backup with $BACKUP_PROG from $DUPLICITY_HOST/$DUPLICITY_PATH/$(hostname)" + LogPrint "========================================================================" + + read -p "ENTER for start restore: " 2>&1 + + export TMPDIR=/mnt/local + + export PYTHONHOME=/usr/lib64/python2.6 + export PYTHONPATH=/usr/lib64/python2.6:/usr/lib64/python2.6/lib-dynload:/usr/lib64/python2.6/site-packages:/usr/lib64/python2.6/site-packages/duplicity + export PASSPHRASE="$BACKUP_DUPLICITY_GPG_ENC_PASSPHRASE" + export HOSTNAME=$(hostname) + + GPG_OPT="$BACKUP_DUPLICITY_GPG_OPTIONS" + GPG_KEY="$BACKUP_DUPLICITY_GPG_ENC_KEY" + PASSPHRASE="$BACKUP_DUPLICITY_GPG_ENC_PASSPHRASE" + + # Setting the pass phrase to decrypt the backup files + export PASSPHRASE + + starttime=$SECONDS + + LogPrint "with CMD: $DUPLICITY_PROG -v 5 $GPG_OPT --encrypt-key $GPG_KEY --force $BACKUP_DUPLICITY_URL/$HOSTNAME/ /mnt/local/" + $DUPLICITY_PROG -v 5 $GPG_OPT --encrypt-key $GPG_KEY --force $BACKUP_DUPLICITY_URL/$HOSTNAME/ /mnt/local | tee $TMP_DIR/duplicity-restore.log + _rc=$? + + transfertime="$((SECONDS-$starttime))" + sleep 1 + + #LogPrint "starttime = $starttime" + #ogPrint "transfertime = $transfertime" + + LogPrint "========================================================================" + + + if [ "$_rc" -gt 0 ]; then + LogPrint "WARNING ! + There was an error while restoring the archive. + Please check '$LOGFILE' and $TMP_DIR/duplicity-restore.log for more information. + You should also manually check the restored system to see wether it is complete. + " + + _message="$(tail -14 ${TMP_DIR}/duplicity-restore.log)" + + LogPrint "Last 14 Lines of ${TMP_DIR}/duplicity-restore.log:" + LogPrint "$_message" + fi + + if [ $_rc -eq 0 ] ; then + LogPrint "Restore comleted in $transfertime seconds." + fi + + LogPrint "========================================================================" + + # Save the logfile to the recoverd filesystem for further checking + LogPrint "Transfering Logfile $TMP_DIR/duplicity-restore.log to /mnt/local/tmp/" + cp -v $TMP_DIR/duplicity-restore.log /mnt/local/tmp/ +fi +