diff --git a/backup-manager b/backup-manager index b2dd115..190eb17 100755 --- a/backup-manager +++ b/backup-manager @@ -188,7 +188,9 @@ debug "Sanitizing the configuration file." source $libdir/sanitize.sh debug "Initializing environment" -bm_init_env +bm_init_env +# Use a single md5 file to store all archives. +export MD5FILE="${BM_REPOSITORY_ROOT}/${BM_ARCHIVE_PREFIX}-hashes.md5" debug "Checking if logger is available" check_logger diff --git a/backup-manager-upload b/backup-manager-upload index 8311fe5..192690e 100755 --- a/backup-manager-upload +++ b/backup-manager-upload @@ -108,6 +108,61 @@ sub get_formated_date($) } } +# The idea behind BM_UPLOADED_ARCHIVES is to have a database of what archives +# have been uploaded so far. This allows multiple execution of upload actions +# within a day without resending all archives of the day from the beginning. + +# Add one file,host pair to $BM_UPLOADED_ARCHIVES database. +# Called immediately *after* successful uploading of an archive. +sub appendto_uploaded_archives($$) +{ + my $file = shift; + my $host = shift; + unless ( defined $file and defined $host ) { + print_error "required args needed"; + return FALSE; + } + + my $upload_fname = $ENV{BM_UPLOADED_ARCHIVES}; + unless ( defined $upload_fname ) { + # Uncomment next line if you want the mandatory use + # of BM_UPLOADED_ARCHIVES (ie always have it around). + #print_error "BM_UPLOADED_ARCHIVES is not defined"; + return FALSE; + } + + # if $file already in database, append host to that line; + # else append a lines "$file $host" to the end. + + my $io_error = 0; + if ( ! system( "grep -q \"^$file \" $upload_fname" ) ) { + my $cmd = "sed -i \"s:^$file .*\$:\& $host:\" $upload_fname"; + $io_error = system("$cmd"); + } + elsif ( open(my $fh, ">>", $upload_fname) ) { + print($fh "$file $host\n") or $io_error = 1; + close $fh; + } + else { + $io_error = 2; + } + if ( $io_error ) { + print_error "IO error: did not update $upload_fname with '$file $host'"; + return FALSE; + } + + return TRUE; +} + +# Get all files of the specified date; filter the list through +# BM_UPLOADED_ARCHIVES if it is set in the environment. +# NOTE: Doing the filtering here implies that the archive is considered +# uploaded if a single upload to a host succeeds; that is even when there +# are failures to other hosts (in case of multiple host uploading). +# To consider it uploaded when all hosts succeed, the filtering must be +# transfered to the individual upload subroutines (and check for existence +# of file,host pair in the database). +# sub get_files_list_from_date($) { my $date = shift; @@ -132,8 +187,21 @@ sub get_files_list_from_date($) exit E_INVALID; } - while (<$g_root_dir/*$date*>) { - push @{$ra_files}, $_; + my $upload_fname = $ENV{BM_UPLOADED_ARCHIVES}; + if ( defined $upload_fname ) { + # filter file list through the BM_UPLOADED_ARCHIVES database + while (<$g_root_dir/*$date*>) { + my $file = $_; + my $cmd = "grep -q '$file' $upload_fname"; + if ( system ("$cmd") ) { + push @{$ra_files}, $file; + } + } + } + else { + while (<$g_root_dir/*$date*>) { + push @{$ra_files}, $_; + } } return $ra_files; @@ -270,7 +338,12 @@ sub send_file_with_scp($$$$$) print_error ("Unable to upload \"$file\". ".($! || $@ || $ret)); return 0; } - return 1; + else { + # use same name in both cases (gpg encryption is done on the fly); + # continue if writing to uploaded archives file fails. + appendto_uploaded_archives($file, $host); + } + return 1; } # How to upload files with scp. @@ -644,22 +717,25 @@ sub send_files_with_ftp($$$$$) # Put all the files over the connexion foreach my $file (@{$ra_files}) { chomp $file; + # continue if writing to uploaded archives file fails. if ($BM_UPLOAD_FTP_SECURE) { - if (!ftptls_put_file ($ftp, $file)) { - print_error "Unable to transfer $file"; - return FALSE; + if (ftptls_put_file ($ftp, $file)) { + appendto_uploaded_archives($file, $host); + print_info "File $file transfered\n"; } else { - print_info "File $file transfered\n"; + print_error "Unable to transfer $file"; + return FALSE; } } else { - if (!ftp_put_file ($ftp, $file)) { - print_error "Unable to transfer $file: " . $ftp->message; - return FALSE; + if (ftp_put_file ($ftp, $file)) { + appendto_uploaded_archives($file, $host); + print_info "File $file transfered\n"; } else { - print_info "File $file transfered\n"; + print_error "Unable to transfer $file: " . $ftp->message; + return FALSE; } } } @@ -847,6 +923,8 @@ sub send_files_with_s3($$$$$$) ); $uploaded{$filename} = $file_length; } + # For the S3 method, we assume success in any case. + appendto_uploaded_archives($file, $host); } # get a list of files and confirm uploads diff --git a/backup-manager.conf.tpl b/backup-manager.conf.tpl index e77e372..b7f0947 100644 --- a/backup-manager.conf.tpl +++ b/backup-manager.conf.tpl @@ -337,6 +337,20 @@ export BM_UPLOAD_HOSTS="" # Where to put archives on the remote hosts (global) export BM_UPLOAD_DESTINATION="" +# Uncomment the 'export ...' line below to activate the uploaded archives +# database. +# Using the database will avoid extraneous uploads to remote hosts in the +# case of running more than one backup-manager jobs per day (such as when +# you are using different configuration files for different parts of your +# filesystem). +# Note that when you upload to multiple hosts, a single succesfull upload +# will mark the archive as uploaded. Thus upload errors to specific hosts +# will have to be resolved manually. +# You can specify any filename, but it is recommended to keep the database +# inside the archive repository. The variable's value has been preset to +# that. +#export BM_UPLOADED_ARCHIVES=${BM_REPOSITORY_ROOT}/${BM_ARCHIVE_PREFIX}-uploaded.list + ############################################################## # The SSH method ############################################################# diff --git a/lib/actions.sh b/lib/actions.sh index e3e56d3..0d10b70 100644 --- a/lib/actions.sh +++ b/lib/actions.sh @@ -52,13 +52,12 @@ function make_archives() esac # Now make sure the md5 file is okay. - md5file="$BM_REPOSITORY_ROOT/${BM_ARCHIVE_PREFIX}-${TODAY}.md5" - if [[ -e $md5file ]] && + if [[ -e $MD5FILE ]] && [[ "$BM_REPOSITORY_SECURE" = "true" ]]; then - chown $BM_REPOSITORY_USER:$BM_REPOSITORY_GROUP $md5file || - warning "Unable to change the owner of \"\$md5file\"." - chmod $BM_ARCHIVE_CHMOD $md5file || - warning "Unable to change file permissions of \"\$md5file\"." + chown $BM_REPOSITORY_USER:$BM_REPOSITORY_GROUP $MD5FILE || + warning "Unable to change the owner of \"\$MD5FILE\"." + chmod $BM_ARCHIVE_CHMOD $MD5FILE || + warning "Unable to change file permissions of \"\$MD5FILE\"." fi done } diff --git a/lib/backup-methods.sh b/lib/backup-methods.sh index 480317d..4658b52 100644 --- a/lib/backup-methods.sh +++ b/lib/backup-methods.sh @@ -38,15 +38,13 @@ function commit_archive() echo "$str ${md5hash})" fi - md5file="$BM_REPOSITORY_ROOT/${BM_ARCHIVE_PREFIX}-${TODAY}.md5" - # Check if the md5file contains already the md5sum of the file_to_create. # In this case, the new md5sum overwrites the old one. - if grep "$base" $md5file >/dev/null 2>&1 ; then - previous_md5sum=$(get_md5sum_from_file $base $md5file) - sed -e "/$base/s/$previous_md5sum/$md5hash/" -i $md5file + if grep "$base" $MD5FILE >/dev/null 2>&1 ; then + previous_md5sum=$(get_md5sum_from_file $base $MD5FILE) + sed -e "/$base/s/$previous_md5sum/$md5hash/" -i $MD5FILE else - echo "$md5hash $base" >> $md5file + echo "$md5hash $base" >> $MD5FILE fi # Now that the file is created, remove previous duplicates if exists... @@ -239,7 +237,9 @@ function __get_flags_relative_blacklist() target="$2" debug "__get_flags_relative_blacklist ($switch, $target)" - target=${target%/} + if [ "$target" != "/" ]; then + target=${target%/} + fi blacklist="" for pattern in $BM_TARBALL_BLACKLIST do @@ -253,7 +253,13 @@ function __get_flags_relative_blacklist() # making a relative path... pattern="${pattern#$target}" length=$(expr length $pattern) - pattern=$(expr substr $pattern 2 $length) + # for $target="/", no spare / is left at the beggining + # after the # substitution; thus take substr from pos 1 + if [ "$target" != "/" ]; then + pattern=$(expr substr $pattern 2 $length) + else + pattern=$(expr substr $pattern 1 $length) + fi # ...and blacklisting it blacklist="$blacklist ${switch}${pattern}" diff --git a/lib/files.sh b/lib/files.sh index 4f34551..820c782 100644 --- a/lib/files.sh +++ b/lib/files.sh @@ -413,10 +413,9 @@ function purge_duplicate_archives() error "Unable to get date from file." # get the md5 hash of the file we parse, in its .md5 file - md5file="$BM_REPOSITORY_ROOT/${BM_ARCHIVE_PREFIX}-${date_of_file}.md5" - md5sum_to_check="$(get_md5sum_from_file $file $md5file)" + md5sum_to_check="$(get_md5sum_from_file $file $MD5FILE)" if [[ -z "$md5sum_to_check" ]]; then - warning "Unable to find the md5 hash of file \"\$file\" in file \"\$md5file\"." + warning "Unable to find the md5 hash of file \"\$file\" in file \"\$MD5FILE\"." continue fi diff --git a/lib/upload-methods.sh b/lib/upload-methods.sh index 8d81feb..3ed31bd 100644 --- a/lib/upload-methods.sh +++ b/lib/upload-methods.sh @@ -64,7 +64,7 @@ function bm_upload_ssh() -h="$bm_upload_hosts" \ -u="$BM_UPLOAD_SSH_USER" \ -d="$BM_UPLOAD_SSH_DESTINATION" \ - -r="$BM_REPOSITORY_ROOT" today 2>$logfile || + -r="$BM_REPOSITORY_ROOT" ${TODAY} 2>$logfile || error "Error reported by backup-manager-upload for method \"scp\", check \"\$logfile\"." rm -f $logfile } @@ -101,7 +101,7 @@ function bm_upload_ssh_gpg() -u="$BM_UPLOAD_SSH_USER" \ -d="$BM_UPLOAD_SSH_DESTINATION" \ -r="$BM_REPOSITORY_ROOT" \ - --gpg-recipient="$BM_UPLOAD_SSHGPG_RECIPIENT" today 2>$logfile|| + --gpg-recipient="$BM_UPLOAD_SSHGPG_RECIPIENT" ${TODAY} 2>$logfile|| error "Error reported by backup-manager-upload for method \"ssh-gpg\", check \"\$logfile\"." rm -f $logfile } @@ -142,7 +142,7 @@ function bm_upload_ftp() -h="$bm_upload_hosts" \ -u="$BM_UPLOAD_FTP_USER" \ -d="$BM_UPLOAD_FTP_DESTINATION" \ - -r="$BM_REPOSITORY_ROOT" today 2>$logfile|| + -r="$BM_REPOSITORY_ROOT" ${TODAY} 2>$logfile|| error "Error reported by backup-manager-upload for method \"ftp\", check \"\$logfile\"." rm -f $logfile @@ -172,7 +172,7 @@ function bm_upload_s3() -h="$bm_upload_hosts" \ -u="$BM_UPLOAD_S3_ACCESS_KEY" \ -b="$BM_UPLOAD_S3_DESTINATION" \ - -r="$BM_REPOSITORY_ROOT" today 2>$logfile || + -r="$BM_REPOSITORY_ROOT" ${TODAY} 2>$logfile || error "Error reported by backup-manager-upload for method \"s3\", check \"\$logfile\"." rm -f $logfile }