Skip to content
Browse files

upload-database patch (debian bts #638920, modified for upstream)

  • Loading branch information...
1 parent a60cafc commit c8f1ca6f18c4c2f45fc40ea875fa072c7d07d3d9 Georgios M. Zarkadas committed
Showing with 103 additions and 11 deletions.
  1. +89 −11 backup-manager-upload
  2. +14 −0 backup-manager.conf.tpl
View
100 backup-manager-upload
@@ -108,6 +108,61 @@ sub get_formated_date($)
}
}
+# The idea behind BM_UPLOADED_ARCHIVES is to have a database of what archives
+# have been uploaded so far. This allows multiple execution of upload actions
+# within a day without resending all archives of the day from the beginning.
+
+# Add one file,host pair to $BM_UPLOADED_ARCHIVES database.
+# Called immediately *after* successful uploading of an archive.
+sub appendto_uploaded_archives($$)
+{
+ my $file = shift;
+ my $host = shift;
+ unless ( defined $file and defined $host ) {
+ print_error "required args needed";
+ return FALSE;
+ }
+
+ my $upload_fname = $ENV{BM_UPLOADED_ARCHIVES};
+ unless ( defined $upload_fname ) {
+ # Uncomment next line if you want the mandatory use
+ # of BM_UPLOADED_ARCHIVES (ie always have it around).
+ #print_error "BM_UPLOADED_ARCHIVES is not defined";
+ return FALSE;
+ }
+
+ # if $file already in database, append host to that line;
+ # else append a lines "$file $host" to the end.
+
+ my $io_error = 0;
+ if ( ! system( "grep -q \"^$file \" $upload_fname" ) ) {
+ my $cmd = "sed -i \"s:^$file .*\$:\& $host:\" $upload_fname";
+ $io_error = system("$cmd");
+ }
+ elsif ( open(my $fh, ">>", $upload_fname) ) {
+ print($fh "$file $host\n") or $io_error = 1;
+ close $fh;
+ }
+ else {
+ $io_error = 2;
+ }
+ if ( $io_error ) {
+ print_error "IO error: did not update $upload_fname with '$file $host'";
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+# Get all files of the specified date; filter the list through
+# BM_UPLOADED_ARCHIVES if it is set in the environment.
+# NOTE: Doing the filtering here implies that the archive is considered
+# uploaded if a single upload to a host succeeds; that is even when there
+# are failures to other hosts (in case of multiple host uploading).
+# To consider it uploaded when all hosts succeed, the filtering must be
+# transfered to the individual upload subroutines (and check for existence
+# of file,host pair in the database).
+#
sub get_files_list_from_date($)
{
my $date = shift;
@@ -132,8 +187,21 @@ sub get_files_list_from_date($)
exit E_INVALID;
}
- while (<$g_root_dir/*$date*>) {
- push @{$ra_files}, $_;
+ my $upload_fname = $ENV{BM_UPLOADED_ARCHIVES};
+ if ( defined $upload_fname ) {
+ # filter file list through the BM_UPLOADED_ARCHIVES database
+ while (<$g_root_dir/*$date*>) {
+ my $file = $_;
+ my $cmd = "grep -q '$file' $upload_fname";
+ if ( system ("$cmd") ) {
+ push @{$ra_files}, $file;
+ }
+ }
+ }
+ else {
+ while (<$g_root_dir/*$date*>) {
+ push @{$ra_files}, $_;
+ }
}
return $ra_files;
@@ -270,7 +338,12 @@ sub send_file_with_scp($$$$$)
print_error ("Unable to upload \"$file\". ".($! || $@ || $ret));
return 0;
}
- return 1;
+ else {
+ # use same name in both cases (gpg encryption is done on the fly);
+ # continue if writing to uploaded archives file fails.
+ appendto_uploaded_archives($file, $host);
+ }
+ return 1;
}
# How to upload files with scp.
@@ -644,22 +717,25 @@ sub send_files_with_ftp($$$$$)
# Put all the files over the connexion
foreach my $file (@{$ra_files}) {
chomp $file;
+ # continue if writing to uploaded archives file fails.
if ($BM_UPLOAD_FTP_SECURE) {
- if (!ftptls_put_file ($ftp, $file)) {
- print_error "Unable to transfer $file";
- return FALSE;
+ if (ftptls_put_file ($ftp, $file)) {
+ appendto_uploaded_archives($file, $host);
+ print_info "File $file transfered\n";
}
else {
- print_info "File $file transfered\n";
+ print_error "Unable to transfer $file";
+ return FALSE;
}
}
else {
- if (!ftp_put_file ($ftp, $file)) {
- print_error "Unable to transfer $file: " . $ftp->message;
- return FALSE;
+ if (ftp_put_file ($ftp, $file)) {
+ appendto_uploaded_archives($file, $host);
+ print_info "File $file transfered\n";
}
else {
- print_info "File $file transfered\n";
+ print_error "Unable to transfer $file: " . $ftp->message;
+ return FALSE;
}
}
}
@@ -847,6 +923,8 @@ sub send_files_with_s3($$$$$$)
);
$uploaded{$filename} = $file_length;
}
+ # For the S3 method, we assume success in any case.
+ appendto_uploaded_archives($file, $host);
}
# get a list of files and confirm uploads
View
14 backup-manager.conf.tpl
@@ -337,6 +337,20 @@ export BM_UPLOAD_HOSTS=""
# Where to put archives on the remote hosts (global)
export BM_UPLOAD_DESTINATION=""
+# Uncomment the 'export ...' line below to activate the uploaded archives
+# database.
+# Using the database will avoid extraneous uploads to remote hosts in the
+# case of running more than one backup-manager jobs per day (such as when
+# you are using different configuration files for different parts of your
+# filesystem).
+# Note that when you upload to multiple hosts, a single succesfull upload
+# will mark the archive as uploaded. Thus upload errors to specific hosts
+# will have to be resolved manually.
+# You can specify any filename, but it is recommended to keep the database
+# inside the archive repository. The variable's value has been preset to
+# that.
+#export BM_UPLOADED_ARCHIVES=${BM_REPOSITORY_ROOT}/${BM_ARCHIVE_PREFIX}-uploaded.list
+
##############################################################
# The SSH method
#############################################################

0 comments on commit c8f1ca6

Please sign in to comment.
Something went wrong with that request. Please try again.