-
-
Notifications
You must be signed in to change notification settings - Fork 279
/
backup
503 lines (458 loc) · 20.7 KB
/
backup
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
#!/bin/bash
CAN_BIND=${CAN_BIND:-1}
# Add a file or a directory to the list of paths to backup
#
# usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory]
# | arg: -s, --src_path= - file or directory to bind or symlink or copy. it shouldn't be in the backup dir.
# | arg: -d, --dest_path= - destination file or directory inside the backup dir
# | arg: -b, --is_big - Indicate data are big (mail, video, image ...)
# | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it.
#
# This helper can be used both in a system backup hook, and in an app backup script
#
# `ynh_backup` writes `src_path` and the relative `dest_path` into a CSV file, and it
# creates the parent destination directory
#
# If `dest_path` is ended by a slash it complete this path with the basename of `src_path`.
#
# Example in the context of a wordpress app :
# ```
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf"
# # => This line will be added into CSV file
# # "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/nginx.conf"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf"
#
# #Deprecated usages (maintained for retro-compatibility)
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "${backup_dir}/conf/nginx.conf"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "/conf/"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf"
#
# ```
#
# How to use `--is_big`:
#
# `--is_big` is used to specify that this part of the backup can be quite huge.
# So, you don't want that your package does backup that part during ynh_backup_before_upgrade.
# In the same way, an user may doesn't want to backup this big part of the app for
# each of his backup. And so handle that part differently.
#
# As this part of your backup may not be done, your restore script has to handle it.
# In your restore script, use `--not_mandatory` with `ynh_restore_file`
# As well in your remove script, you should not remove those data ! Or an user may end up with
# a failed upgrade restoring an app without data anymore !
#
# To have the benefit of `--is_big` while doing a backup, you can whether set the environement
# variable `BACKUP_CORE_ONLY` to 1 (`BACKUP_CORE_ONLY=1`) before the backup command. It will affect
# only that backup command.
# Or set the config `do_not_backup_data` to 1 into the `settings.yml` of the app. This will affect
# all backups for this app until the setting is removed.
#
# Requires YunoHost version 2.4.0 or higher.
# Requires YunoHost version 3.5.0 or higher for the argument `--not_mandatory`
ynh_backup() {
# TODO find a way to avoid injection by file strange naming !
# Declare an array to define the options of this helper.
local legacy_args=sdbm
local -A args_array=([s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory)
local src_path
local dest_path
local is_big
local not_mandatory
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
dest_path="${dest_path:-}"
is_big="${is_big:-0}"
not_mandatory="${not_mandatory:-0}"
BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0}
test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --app=$app --key=do_not_backup_data)
# If backing up core only (used by ynh_backup_before_upgrade),
# don't backup big data items
if [ $is_big -eq 1 ] && ([ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ]); then
if [ $BACKUP_CORE_ONLY -eq 1 ]; then
ynh_print_info --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set."
else
ynh_print_info --message="$src_path will not be saved, because 'do_not_backup_data' is set."
fi
return 0
fi
# ==============================================================================
# Format correctly source and destination paths
# ==============================================================================
# Be sure the source path is not empty
if [ ! -e "$src_path" ]; then
ynh_print_warn --message="Source path '${src_path}' does not exist"
if [ "$not_mandatory" == "0" ]; then
# This is a temporary fix for fail2ban config files missing after the migration to stretch.
if echo "${src_path}" | grep --quiet "/etc/fail2ban"; then
touch "${src_path}"
ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!"
else
return 1
fi
else
return 0
fi
fi
# Transform the source path as an absolute path
# If it's a dir remove the ending /
src_path=$(realpath "$src_path")
# If there is no destination path, initialize it with the source path
# relative to "/".
# eg: src_path=/etc/yunohost -> dest_path=etc/yunohost
if [[ -z "$dest_path" ]]; then
dest_path="${src_path#/}"
else
if [[ "${dest_path:0:1}" == "/" ]]; then
# If the destination path is an absolute path, transform it as a path
# relative to the current working directory ($YNH_CWD)
#
# If it's an app backup script that run this helper, YNH_CWD is equal to
# $YNH_BACKUP_DIR/apps/APP_INSTANCE_NAME/backup/
#
# If it's a system part backup script, YNH_CWD is equal to $YNH_BACKUP_DIR
dest_path="${dest_path#$YNH_CWD/}"
# Case where $2 is an absolute dir but doesn't begin with $YNH_CWD
if [[ "${dest_path:0:1}" == "/" ]]; then
dest_path="${dest_path#/}"
fi
fi
# Complete dest_path if ended by a /
if [[ "${dest_path: -1}" == "/" ]]; then
dest_path="${dest_path}/$(basename $src_path)"
fi
fi
# Check if dest_path already exists in tmp archive
if [[ -e "${dest_path}" ]]; then
ynh_print_err --message="Destination path '${dest_path}' already exist"
return 1
fi
# Add the relative current working directory to the destination path
local rel_dir="${YNH_CWD#$YNH_BACKUP_DIR}"
rel_dir="${rel_dir%/}/"
dest_path="${rel_dir}${dest_path}"
dest_path="${dest_path#/}"
# ==============================================================================
# ==============================================================================
# Write file to backup into backup_list
# ==============================================================================
local src=$(echo "${src_path}" | sed --regexp-extended 's/"/\"\"/g')
local dest=$(echo "${dest_path}" | sed --regexp-extended 's/"/\"\"/g')
echo "\"${src}\",\"${dest}\"" >>"${YNH_BACKUP_CSV}"
# ==============================================================================
# Create the parent dir of the destination path
# It's for retro compatibility, some script consider ynh_backup creates this dir
mkdir --parents $(dirname "$YNH_BACKUP_DIR/${dest_path}")
}
# Restore all files that were previously backuped in a core backup script or app backup script
#
# usage: ynh_restore
#
# Requires YunoHost version 2.6.4 or higher.
ynh_restore() {
# Deduce the relative path of $YNH_CWD
local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}"
REL_DIR="${REL_DIR%/}/"
# For each destination path begining by $REL_DIR
cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \
| while read line; do
local ORIGIN_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\".*\"$)")
local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)")
ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH"
done
}
# Return the path in the archive where has been stocked the origin path
#
# [internal]
#
# usage: _get_archive_path ORIGIN_PATH
_get_archive_path() {
# For security reasons we use csv python library to read the CSV
python3 -c "
import sys
import csv
with open(sys.argv[1], 'r') as backup_file:
backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest'])
for row in backup_csv:
if row['source']==sys.argv[2].strip('\"'):
print(row['dest'])
sys.exit(0)
raise Exception('Original path for %s not found' % sys.argv[2])
" "${YNH_BACKUP_CSV}" "$1"
return $?
}
# Restore a file or a directory
#
# usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] [--not_mandatory]
# | arg: -o, --origin_path= - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive
# | arg: -d, --dest_path= - Path where restore the file or the dir. If unspecified, the destination will be `ORIGIN_PATH` or if the `ORIGIN_PATH` doesn't exist in the archive, the destination will be searched into `backup.csv`
# | arg: -m, --not_mandatory - Indicate that if the file is missing, the restore process can ignore it.
#
# Use the registered path in backup_list by ynh_backup to restore the file at the right place.
#
# examples:
# ynh_restore_file -o "/etc/nginx/conf.d/$domain.d/$app.conf"
# # You can also use relative paths:
# ynh_restore_file -o "conf/nginx.conf"
#
# If `DEST_PATH` already exists and is lighter than 500 Mo, a backup will be made in
# `/var/cache/yunohost/appconfbackup/`. Otherwise, the existing file is removed.
#
# if `apps/$app/etc/nginx/conf.d/$domain.d/$app.conf` exists, restore it into
# `/etc/nginx/conf.d/$domain.d/$app.conf`
# if no, search for a match in the csv (eg: conf/nginx.conf) and restore it into
# `/etc/nginx/conf.d/$domain.d/$app.conf`
#
# Requires YunoHost version 2.6.4 or higher.
# Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory
ynh_restore_file() {
# Declare an array to define the options of this helper.
local legacy_args=odm
local -A args_array=([o]=origin_path= [d]=dest_path= [m]=not_mandatory)
local origin_path
local dest_path
local not_mandatory
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
origin_path="/${origin_path#/}"
# Default value for dest_path = /$origin_path
dest_path="${dest_path:-$origin_path}"
not_mandatory="${not_mandatory:-0}"
local archive_path="$YNH_CWD${origin_path}"
# If archive_path doesn't exist, search for a corresponding path in CSV
if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then
if [ "$not_mandatory" == "0" ]; then
archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")"
else
return 0
fi
fi
# Move the old directory if it already exists
if [[ -e "${dest_path}" ]]; then
# Check if the file/dir size is less than 500 Mo
if [[ $(du --summarize --bytes ${dest_path} | cut --delimiter="/" --fields=1) -le "500000000" ]]; then
local backup_file="/var/cache/yunohost/appconfbackup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')"
mkdir --parents "$(dirname "$backup_file")"
mv "${dest_path}" "$backup_file" # Move the current file or directory
else
ynh_secure_remove --file=${dest_path}
fi
fi
# Restore origin_path into dest_path
mkdir --parents $(dirname "$dest_path")
# Do a copy if it's just a mounting point
if mountpoint --quiet $YNH_BACKUP_DIR; then
if [[ -d "${archive_path}" ]]; then
archive_path="${archive_path}/."
mkdir --parents "$dest_path"
fi
cp --archive "$archive_path" "${dest_path}"
# Do a move if YNH_BACKUP_DIR is already a copy
else
mv "$archive_path" "${dest_path}"
fi
# Boring hack for nginx conf file mapped to php7.3
# Note that there's no need to patch the fpm config because most php apps
# will call "ynh_add_fpm_config" during restore, effectively recreating the file from scratch
if [[ "${dest_path}" == "/etc/nginx/conf.d/"* ]] && grep 'php7.3.*sock' "${dest_path}"
then
sed -i 's/php7.3/php7.4/g' "${dest_path}"
fi
}
# Calculate and store a file checksum into the app settings
#
# usage: ynh_store_file_checksum --file=file
# | arg: -f, --file= - The file on which the checksum will performed, then stored.
#
# $app should be defined when calling this helper
#
# Requires YunoHost version 2.6.4 or higher.
ynh_store_file_checksum() {
# Declare an array to define the options of this helper.
local legacy_args=f
local -A args_array=([f]=file= [u]=update_only)
local file
local update_only
update_only="${update_only:-0}"
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
# If update only, we don't save the new checksum if no old checksum exist
if [ $update_only -eq 1 ]; then
local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
if [ -z "${checksum_value}" ]; then
unset backup_file_checksum
return 0
fi
fi
ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1)
if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
# Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ...
local file_path_base64=$(echo "$file" | base64 -w0)
mkdir -p /var/cache/yunohost/appconfbackup/
cat $file > /var/cache/yunohost/appconfbackup/original_${file_path_base64}
fi
# If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup
if [ -n "${backup_file_checksum-}" ]; then
# Print the diff between the previous file and the new one.
# diff return 1 if the files are different, so the || true
diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true
fi
# Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it.
unset backup_file_checksum
}
# Verify the checksum and backup the file if it's different
#
# usage: ynh_backup_if_checksum_is_different --file=file
# | arg: -f, --file= - The file on which the checksum test will be perfomed.
# | ret: the name of a backup file, or nothing
#
# This helper is primarily meant to allow to easily backup personalised/manually
# modified config files.
#
# Requires YunoHost version 2.6.4 or higher.
ynh_backup_if_checksum_is_different() {
# Declare an array to define the options of this helper.
local legacy_args=f
local -A args_array=([f]=file=)
local file
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
# backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum
backup_file_checksum=""
if [ -n "$checksum_value" ]; then # Proceed only if a value was stored into the app settings
if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status; then # If the checksum is now different
backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
mkdir --parents "$(dirname "$backup_file_checksum")"
cp --archive "$file" "$backup_file_checksum" # Backup the current file
ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum"
echo "$backup_file_checksum" # Return the name of the backup file
if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
local file_path_base64=$(echo "$file" | base64 -w0)
if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64}
then
ynh_print_warn "Diff with the original file:"
diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true
fi
fi
fi
fi
}
# Delete a file checksum from the app settings
#
# usage: ynh_delete_file_checksum --file=file
# | arg: -f, --file= - The file for which the checksum will be deleted
#
# $app should be defined when calling this helper
#
# Requires YunoHost version 3.3.1 or higher.
ynh_delete_file_checksum() {
# Declare an array to define the options of this helper.
local legacy_args=f
local -A args_array=([f]=file=)
local file
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
ynh_app_setting_delete --app=$app --key=$checksum_setting_name
}
# Checks a backup archive exists
#
# [internal]
#
ynh_backup_archive_exists() {
yunohost backup list --output-as json --quiet \
| jq -e --arg archive "$1" '.archives | index($archive)' >/dev/null
}
# Make a backup in case of failed upgrade
#
# usage: ynh_backup_before_upgrade
#
# Usage in a package script:
# ```
# ynh_backup_before_upgrade
# ynh_clean_setup () {
# ynh_restore_upgradebackup
# }
# ynh_abort_if_errors
# ```
#
# Requires YunoHost version 2.7.2 or higher.
ynh_backup_before_upgrade() {
if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]; then
ynh_print_warn --message="This app doesn't have any backup script."
return
fi
backup_number=1
local old_backup_number=2
local app_bck=${app//_/-} # Replace all '_' by '-'
NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then
# Check if a backup already exists with the prefix 1
if ynh_backup_archive_exists "$app_bck-pre-upgrade1"; then
# Prefix becomes 2 to preserve the previous backup
backup_number=2
old_backup_number=1
fi
# Create backup
BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug
if [ "$?" -eq 0 ]; then
# If the backup succeeded, remove the previous backup
if ynh_backup_archive_exists "$app_bck-pre-upgrade$old_backup_number"; then
# Remove the previous backup only if it exists
yunohost backup delete $app_bck-pre-upgrade$old_backup_number >/dev/null
fi
else
ynh_die --message="Backup failed, the upgrade process was aborted."
fi
else
ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup"
fi
}
# Restore a previous backup if the upgrade process failed
#
# usage: ynh_restore_upgradebackup
#
# Usage in a package script:
# ```
# ynh_backup_before_upgrade
# ynh_clean_setup () {
# ynh_restore_upgradebackup
# }
# ynh_abort_if_errors
# ```
#
# Requires YunoHost version 2.7.2 or higher.
ynh_restore_upgradebackup() {
ynh_print_err --message="Upgrade failed."
local app_bck=${app//_/-} # Replace all '_' by '-'
NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then
# Check if an existing backup can be found before removing and restoring the application.
if ynh_backup_archive_exists "$app_bck-pre-upgrade$backup_number"; then
# Remove the application then restore it
yunohost app remove $app
# Restore the backup
yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug
if [[ -d /etc/yunohost/apps/$app ]]
then
ynh_die --message="The app was restored to the way it was before the failed upgrade."
else
ynh_die --message="Uhoh ... Yunohost failed to restore the app to the way it was before the failed upgrade :|"
fi
fi
else
ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !"
fi
}