backup

#!/bin/bash

CAN_BIND=${CAN_BIND:-1}

# Add a file or a directory to the list of paths to backup
#
# usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory]
# | arg: -s, --src_path=        - file or directory to bind or symlink or copy. it shouldn't be in the backup dir.
# | arg: -d, --dest_path=       - destination file or directory inside the backup dir
# | arg: -b, --is_big           - Indicate data are big (mail, video, image ...)
# | arg: -m, --not_mandatory    - Indicate that if the file is missing, the backup can ignore it.
#
# This helper can be used both in a system backup hook, and in an app backup script
#
# `ynh_backup` writes `src_path` and the relative `dest_path` into a CSV file, and it
# creates the parent destination directory
#
# If `dest_path` is ended by a slash it complete this path with the basename of `src_path`.
#
# Example in the context of a wordpress app :
# ```
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf"
# # => This line will be added into CSV file
# # "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/nginx.conf"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf"
#
# #Deprecated usages (maintained for retro-compatibility)
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "${backup_dir}/conf/nginx.conf"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "/conf/"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf"
#
# ```
#
# How to use `--is_big`:
#
# `--is_big` is used to specify that this part of the backup can be quite huge.
# So, you don't want that your package does backup that part during ynh_backup_before_upgrade.
# In the same way, an user may doesn't want to backup this big part of the app for
# each of his backup. And so handle that part differently.
#
# As this part of your backup may not be done, your restore script has to handle it.
# In your restore script, use `--not_mandatory` with `ynh_restore_file`
# As well in your remove script, you should not remove those data ! Or an user may end up with
# a failed upgrade restoring an app without data anymore !
#
# To have the benefit of `--is_big` while doing a backup, you can whether set the environement
# variable `BACKUP_CORE_ONLY` to 1 (`BACKUP_CORE_ONLY=1`) before the backup command. It will affect
# only that backup command.
# Or set the config `do_not_backup_data` to 1 into the `settings.yml` of the app. This will affect
# all backups for this app until the setting is removed.
#
# Requires YunoHost version 2.4.0 or higher.
# Requires YunoHost version 3.5.0 or higher for the argument `--not_mandatory`
ynh_backup() {
    # TODO find a way to avoid injection by file strange naming !

    # Declare an array to define the options of this helper.
    local legacy_args=sdbm
    local -A args_array=([s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory)
    local src_path
    local dest_path
    local is_big
    local not_mandatory
    # Manage arguments with getopts
    ynh_handle_getopts_args "$@"
    dest_path="${dest_path:-}"
    is_big="${is_big:-0}"
    not_mandatory="${not_mandatory:-0}"

    BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0}
    test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --app=$app --key=do_not_backup_data)

    # If backing up core only (used by ynh_backup_before_upgrade),
    # don't backup big data items
    if [ $is_big -eq 1 ] && ([ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ]); then
        if [ $BACKUP_CORE_ONLY -eq 1 ]; then
            ynh_print_info --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set."
        else
            ynh_print_info --message="$src_path will not be saved, because 'do_not_backup_data' is set."
        fi
        return 0
    fi

    # ==============================================================================
    # Format correctly source and destination paths
    # ==============================================================================
    # Be sure the source path is not empty
    if [ ! -e "$src_path" ]; then
        ynh_print_warn --message="Source path '${src_path}' does not exist"
        if [ "$not_mandatory" == "0" ]; then
            # This is a temporary fix for fail2ban config files missing after the migration to stretch.
            if echo "${src_path}" | grep --quiet "/etc/fail2ban"; then
                touch "${src_path}"
                ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!"
            else
                return 1
            fi
        else
            return 0
        fi
    fi

    # Transform the source path as an absolute path
    # If it's a dir remove the ending /
    src_path=$(realpath "$src_path")

    # If there is no destination path, initialize it with the source path
    # relative to "/".
    # eg: src_path=/etc/yunohost -> dest_path=etc/yunohost
    if [[ -z "$dest_path" ]]; then
        dest_path="${src_path#/}"

    else
        if [[ "${dest_path:0:1}" == "/" ]]; then

            # If the destination path is an absolute path, transform it as a path
            # relative to the current working directory ($YNH_CWD)
            #
            # If it's an app backup script that run this helper, YNH_CWD is equal to
            # $YNH_BACKUP_DIR/apps/APP_INSTANCE_NAME/backup/
            #
            # If it's a system part backup script, YNH_CWD is equal to $YNH_BACKUP_DIR
            dest_path="${dest_path#$YNH_CWD/}"

            # Case where $2 is an absolute dir but doesn't begin with $YNH_CWD
            if [[ "${dest_path:0:1}" == "/" ]]; then
                dest_path="${dest_path#/}"
            fi
        fi

        # Complete dest_path if ended by a /
        if [[ "${dest_path: -1}" == "/" ]]; then
            dest_path="${dest_path}/$(basename $src_path)"
        fi
    fi

    # Check if dest_path already exists in tmp archive
    if [[ -e "${dest_path}" ]]; then
        ynh_print_err --message="Destination path '${dest_path}' already exist"
        return 1
    fi

    # Add the relative current working directory to the destination path
    local rel_dir="${YNH_CWD#$YNH_BACKUP_DIR}"
    rel_dir="${rel_dir%/}/"
    dest_path="${rel_dir}${dest_path}"
    dest_path="${dest_path#/}"
    # ==============================================================================

    # ==============================================================================
    # Write file to backup into backup_list
    # ==============================================================================
    local src=$(echo "${src_path}" | sed --regexp-extended 's/"/\"\"/g')
    local dest=$(echo "${dest_path}" | sed --regexp-extended 's/"/\"\"/g')
    echo "\"${src}\",\"${dest}\"" >>"${YNH_BACKUP_CSV}"

    # ==============================================================================

    # Create the parent dir of the destination path
    # It's for retro compatibility, some script consider ynh_backup creates this dir
    mkdir --parents $(dirname "$YNH_BACKUP_DIR/${dest_path}")
}

# Restore all files that were previously backuped in a core backup script or app backup script
#
# usage: ynh_restore
#
# Requires YunoHost version 2.6.4 or higher.
ynh_restore() {
    # Deduce the relative path of $YNH_CWD
    local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}"
    REL_DIR="${REL_DIR%/}/"

    # For each destination path begining by $REL_DIR
    cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \
        | while read line; do
            local ORIGIN_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\".*\"$)")
            local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)")
            ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH"
        done
}

# Return the path in the archive where has been stocked the origin path
#
# [internal]
#
# usage: _get_archive_path ORIGIN_PATH
_get_archive_path() {
    # For security reasons we use csv python library to read the CSV
    python3 -c "
import sys
import csv
with open(sys.argv[1], 'r') as backup_file:
    backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest'])
    for row in backup_csv:
        if row['source']==sys.argv[2].strip('\"'):
            print(row['dest'])
            sys.exit(0)
    raise Exception('Original path for %s not found' % sys.argv[2])
    " "${YNH_BACKUP_CSV}" "$1"
    return $?
}

# Restore a file or a directory
#
# usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] [--not_mandatory]
# | arg: -o, --origin_path=     - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive
# | arg: -d, --dest_path=       - Path where restore the file or the dir. If unspecified, the destination will be `ORIGIN_PATH` or if the `ORIGIN_PATH` doesn't exist in the archive, the destination will be searched into `backup.csv`
# | arg: -m, --not_mandatory    - Indicate that if the file is missing, the restore process can ignore it.
#
# Use the registered path in backup_list by ynh_backup to restore the file at the right place.
#
# examples:
#     ynh_restore_file -o "/etc/nginx/conf.d/$domain.d/$app.conf"
#     # You can also use relative paths:
#     ynh_restore_file -o "conf/nginx.conf"
#
# If `DEST_PATH` already exists and is lighter than 500 Mo, a backup will be made in
# `/var/cache/yunohost/appconfbackup/`. Otherwise, the existing file is removed.
#
# if `apps/$app/etc/nginx/conf.d/$domain.d/$app.conf` exists, restore it into
# `/etc/nginx/conf.d/$domain.d/$app.conf`
# if no, search for a match in the csv (eg: conf/nginx.conf) and restore it into
# `/etc/nginx/conf.d/$domain.d/$app.conf`
#
# Requires YunoHost version 2.6.4 or higher.
# Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory
ynh_restore_file() {
    # Declare an array to define the options of this helper.
    local legacy_args=odm
    local -A args_array=([o]=origin_path= [d]=dest_path= [m]=not_mandatory)
    local origin_path
    local dest_path
    local not_mandatory
    # Manage arguments with getopts
    ynh_handle_getopts_args "$@"
    origin_path="/${origin_path#/}"
    # Default value for dest_path = /$origin_path
    dest_path="${dest_path:-$origin_path}"
    not_mandatory="${not_mandatory:-0}"

    local archive_path="$YNH_CWD${origin_path}"
    # If archive_path doesn't exist, search for a corresponding path in CSV
    if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then
        if [ "$not_mandatory" == "0" ]; then
            archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")"
        else
            return 0
        fi
    fi

    # Move the old directory if it already exists
    if [[ -e "${dest_path}" ]]; then
        # Check if the file/dir size is less than 500 Mo
        if [[ $(du --summarize --bytes ${dest_path} | cut --delimiter="/" --fields=1) -le "500000000" ]]; then
            local backup_file="/var/cache/yunohost/appconfbackup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')"
            mkdir --parents "$(dirname "$backup_file")"
            mv "${dest_path}" "$backup_file" # Move the current file or directory
        else
            ynh_secure_remove --file=${dest_path}
        fi
    fi

    # Restore origin_path into dest_path
    mkdir --parents $(dirname "$dest_path")

    # Do a copy if it's just a mounting point
    if mountpoint --quiet $YNH_BACKUP_DIR; then
        if [[ -d "${archive_path}" ]]; then
            archive_path="${archive_path}/."
            mkdir --parents "$dest_path"
        fi
        cp --archive "$archive_path" "${dest_path}"
    # Do a move if YNH_BACKUP_DIR is already a copy
    else
        mv "$archive_path" "${dest_path}"
    fi

    # Boring hack for nginx conf file mapped to php7.3
    # Note that there's no need to patch the fpm config because most php apps
    # will call "ynh_add_fpm_config" during restore, effectively recreating the file from scratch
    if [[ "${dest_path}" == "/etc/nginx/conf.d/"* ]] && grep 'php7.3.*sock' "${dest_path}"
    then
        sed -i 's/php7.3/php7.4/g' "${dest_path}"
    fi
}

# Calculate and store a file checksum into the app settings
#
# usage: ynh_store_file_checksum --file=file
# | arg: -f, --file=    - The file on which the checksum will performed, then stored.
#
# $app should be defined when calling this helper
#
# Requires YunoHost version 2.6.4 or higher.
ynh_store_file_checksum() {
    # Declare an array to define the options of this helper.
    local legacy_args=f
    local -A args_array=([f]=file= [u]=update_only)
    local file
    local update_only
    update_only="${update_only:-0}"

    # Manage arguments with getopts
    ynh_handle_getopts_args "$@"

    local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'

    # If update only, we don't save the new checksum if no old checksum exist
    if [ $update_only -eq 1 ]; then
        local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
        if [ -z "${checksum_value}" ]; then
            unset backup_file_checksum
            return 0
        fi
    fi

    ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1)

    if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
        # Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ...
        local file_path_base64=$(echo "$file" | base64 -w0)
        mkdir -p /var/cache/yunohost/appconfbackup/
        cat $file > /var/cache/yunohost/appconfbackup/original_${file_path_base64}
    fi

    # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup
    if [ -n "${backup_file_checksum-}" ]; then
        # Print the diff between the previous file and the new one.
        # diff return 1 if the files are different, so the || true
        diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true
    fi
    # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it.
    unset backup_file_checksum
}

# Verify the checksum and backup the file if it's different
#
# usage: ynh_backup_if_checksum_is_different --file=file
# | arg: -f, --file=    - The file on which the checksum test will be perfomed.
# | ret: the name of a backup file, or nothing
#
# This helper is primarily meant to allow to easily backup personalised/manually
# modified config files.
#
# Requires YunoHost version 2.6.4 or higher.
ynh_backup_if_checksum_is_different() {
    # Declare an array to define the options of this helper.
    local legacy_args=f
    local -A args_array=([f]=file=)
    local file
    # Manage arguments with getopts
    ynh_handle_getopts_args "$@"

    local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
    local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
    # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum
    backup_file_checksum=""
    if [ -n "$checksum_value" ]; then                                                     # Proceed only if a value was stored into the app settings
        if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status; then # If the checksum is now different

            backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
            mkdir --parents "$(dirname "$backup_file_checksum")"
            cp --archive "$file" "$backup_file_checksum" # Backup the current file
            ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum"
            echo "$backup_file_checksum" # Return the name of the backup file
            if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
                local file_path_base64=$(echo "$file" | base64 -w0)
                if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64}
                then
                    ynh_print_warn "Diff with the original file:"
                    diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true
                fi
            fi
        fi
    fi
}

# Delete a file checksum from the app settings
#
# usage: ynh_delete_file_checksum --file=file
# | arg: -f, --file=    - The file for which the checksum will be deleted
#
# $app should be defined when calling this helper
#
# Requires YunoHost version 3.3.1 or higher.
ynh_delete_file_checksum() {
    # Declare an array to define the options of this helper.
    local legacy_args=f
    local -A args_array=([f]=file=)
    local file
    # Manage arguments with getopts
    ynh_handle_getopts_args "$@"

    local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
    ynh_app_setting_delete --app=$app --key=$checksum_setting_name
}

# Checks a backup archive exists
#
# [internal]
#
ynh_backup_archive_exists() {
    yunohost backup list --output-as json --quiet \
        | jq -e --arg archive "$1" '.archives | index($archive)' >/dev/null
}

# Make a backup in case of failed upgrade
#
# usage: ynh_backup_before_upgrade
#
# Usage in a package script:
# ```
#   ynh_backup_before_upgrade
#   ynh_clean_setup () {
#       ynh_restore_upgradebackup
#   }
#   ynh_abort_if_errors
# ```
#
# Requires YunoHost version 2.7.2 or higher.
ynh_backup_before_upgrade() {
    if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]; then
        ynh_print_warn --message="This app doesn't have any backup script."
        return
    fi
    backup_number=1
    local old_backup_number=2
    local app_bck=${app//_/-} # Replace all '_' by '-'
    NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}

    if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then
        # Check if a backup already exists with the prefix 1
        if ynh_backup_archive_exists "$app_bck-pre-upgrade1"; then
            # Prefix becomes 2 to preserve the previous backup
            backup_number=2
            old_backup_number=1
        fi

        # Create backup
        BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug
        if [ "$?" -eq 0 ]; then
            # If the backup succeeded, remove the previous backup
            if ynh_backup_archive_exists "$app_bck-pre-upgrade$old_backup_number"; then
                # Remove the previous backup only if it exists
                yunohost backup delete $app_bck-pre-upgrade$old_backup_number >/dev/null
            fi
        else
            ynh_die --message="Backup failed, the upgrade process was aborted."
        fi
    else
        ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup"
    fi
}

# Restore a previous backup if the upgrade process failed
#
# usage: ynh_restore_upgradebackup
#
# Usage in a package script:
# ```
#   ynh_backup_before_upgrade
#   ynh_clean_setup () {
#       ynh_restore_upgradebackup
#   }
#   ynh_abort_if_errors
# ```
#
# Requires YunoHost version 2.7.2 or higher.
ynh_restore_upgradebackup() {
    ynh_print_err --message="Upgrade failed."
    local app_bck=${app//_/-} # Replace all '_' by '-'

    NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}

    if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then
        # Check if an existing backup can be found before removing and restoring the application.
        if ynh_backup_archive_exists "$app_bck-pre-upgrade$backup_number"; then
            # Remove the application then restore it
            yunohost app remove $app
            # Restore the backup
            yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug
            if [[ -d /etc/yunohost/apps/$app ]]
            then
                ynh_die --message="The app was restored to the way it was before the failed upgrade."
            else
                ynh_die --message="Uhoh ... Yunohost failed to restore the app to the way it was before the failed upgrade :|"
            fi
        fi
    else
        ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !"
    fi
}