Merge pull request #14 from gzarkadas/master

Four independent patches
sukria · Oct 13, 2011 · 72b93f0 · 72b93f0
2 parents a60cafc + a235071
commit 72b93f0
Show file tree

Hide file tree

Showing 7 changed files with 131 additions and 33 deletions.
diff --git a/backup-manager b/backup-manager
@@ -188,7 +188,9 @@ debug "Sanitizing the configuration file."
 source $libdir/sanitize.sh
 
 debug "Initializing environment"
-bm_init_env 
+bm_init_env
+# Use a single md5 file to store all archives.
+export MD5FILE="${BM_REPOSITORY_ROOT}/${BM_ARCHIVE_PREFIX}-hashes.md5"
 
 debug "Checking if logger is available"
 check_logger

diff --git a/backup-manager-upload b/backup-manager-upload
@@ -108,6 +108,61 @@ sub get_formated_date($)
 	}
 }
 
+# The idea behind BM_UPLOADED_ARCHIVES is to have a database of what archives
+# have been uploaded so far. This allows multiple execution of upload actions
+# within a day without resending all archives of the day from the beginning.
+
+# Add one file,host pair to $BM_UPLOADED_ARCHIVES database.
+# Called immediately *after* successful uploading of an archive.
+sub appendto_uploaded_archives($$)
+{
+    my $file = shift;
+    my $host = shift;
+    unless ( defined $file and defined $host ) {
+        print_error "required args needed";
+        return FALSE;
+    }
+
+    my $upload_fname = $ENV{BM_UPLOADED_ARCHIVES};
+    unless ( defined $upload_fname ) {
+        # Uncomment next line if you want the mandatory use
+        # of BM_UPLOADED_ARCHIVES (ie always have it around).
+        #print_error "BM_UPLOADED_ARCHIVES is not defined";
+        return FALSE;
+    }
+
+    # if $file already in database, append host to that line;
+    # else append a lines "$file $host" to the end.
+
+    my $io_error = 0;
+    if ( ! system( "grep -q \"^$file \" $upload_fname" ) ) {
+        my $cmd = "sed -i \"s:^$file .*\$:\& $host:\" $upload_fname";
+        $io_error = system("$cmd");
+    }
+    elsif ( open(my $fh, ">>", $upload_fname) ) {
+        print($fh "$file $host\n") or $io_error = 1;
+        close $fh;
+    }
+    else {
+        $io_error = 2;
+    }
+    if ( $io_error ) {
+        print_error "IO error: did not update $upload_fname with '$file $host'";
+        return FALSE;
+    }
+
+    return TRUE;
+}
+
+# Get all files of the specified date; filter the list through 
+# BM_UPLOADED_ARCHIVES if it is set in the environment.
+# NOTE:  Doing the filtering here implies that the archive is considered
+# uploaded if a single upload to a host succeeds; that is even when there
+# are failures to other hosts (in case of multiple host uploading).
+# To consider it uploaded when all hosts succeed, the filtering must be
+# transfered to the individual upload subroutines (and check for existence
+# of file,host pair in the database).
+#
 sub get_files_list_from_date($)
 {
 	my $date = shift;
@@ -132,8 +187,21 @@ sub get_files_list_from_date($)
         exit E_INVALID;
     }
 
-	while (<$g_root_dir/*$date*>) {
-        push @{$ra_files}, $_;
+    my $upload_fname = $ENV{BM_UPLOADED_ARCHIVES};
+    if ( defined $upload_fname ) {
+        # filter file list through the BM_UPLOADED_ARCHIVES database
+    	while (<$g_root_dir/*$date*>) {
+            my $file = $_;
+            my $cmd = "grep -q '$file' $upload_fname";
+            if ( system ("$cmd") ) {
+                push @{$ra_files}, $file;
+            }
+        }
+    }
+    else {
+    	while (<$g_root_dir/*$date*>) {
+            push @{$ra_files}, $_;
+        }
 	}
 
 	return $ra_files;
@@ -270,7 +338,12 @@ sub send_file_with_scp($$$$$)
 		print_error ("Unable to upload \"$file\". ".($! || $@ || $ret));
         return 0;
 	}
-	return 1;
+    else {
+        # use same name in both cases (gpg encryption is done on the fly);
+        # continue if writing to uploaded archives file fails.
+        appendto_uploaded_archives($file, $host);
+    }
+    return 1;
 }
 
 # How to upload files with scp.
@@ -644,22 +717,25 @@ sub send_files_with_ftp($$$$$)
         # Put all the files over the connexion
         foreach my $file (@{$ra_files}) {
             chomp $file;
+            # continue if writing to uploaded archives file fails.
             if ($BM_UPLOAD_FTP_SECURE) {
-                if (!ftptls_put_file ($ftp, $file)) {
-		    print_error "Unable to transfer $file";
-		    return FALSE;
+                if (ftptls_put_file ($ftp, $file)) {
+                    appendto_uploaded_archives($file, $host);
+                    print_info "File $file transfered\n";
                 }
                 else {
-			print_info "File $file transfered\n";
+                    print_error "Unable to transfer $file";
+                    return FALSE;
                 }
             }
             else {
-                if (!ftp_put_file ($ftp, $file)) {
-		    print_error "Unable to transfer $file: " . $ftp->message;
-		    return FALSE;
+                if (ftp_put_file ($ftp, $file)) {
+                    appendto_uploaded_archives($file, $host);
+                    print_info "File $file transfered\n";
                 }
                 else {
-            print_info "File $file transfered\n";
+                    print_error "Unable to transfer $file: " . $ftp->message;
+                    return FALSE;
                 }
             }
         }
@@ -847,6 +923,8 @@ sub send_files_with_s3($$$$$$)
 				);
 				$uploaded{$filename} = $file_length;
 			}
+            # For the S3 method, we assume success in any case.
+            appendto_uploaded_archives($file, $host);
 		}
 
 		# get a list of files and confirm uploads

diff --git a/backup-manager.conf.tpl b/backup-manager.conf.tpl
@@ -337,6 +337,20 @@ export BM_UPLOAD_HOSTS=""
 # Where to put archives on the remote hosts (global)
 export BM_UPLOAD_DESTINATION=""
 
+# Uncomment the 'export ...' line below to activate the uploaded archives
+# database.
+# Using the database will avoid extraneous uploads to remote hosts in the
+# case of running more than one backup-manager jobs per day (such as when
+# you are using different configuration files for different parts of your
+# filesystem).
+# Note that when you upload to multiple hosts, a single succesfull upload
+# will mark the archive as uploaded. Thus upload errors to specific hosts
+# will have to be resolved manually.
+# You can specify any filename, but it is recommended to keep the database
+# inside the archive repository. The variable's value has been preset to
+# that.
+#export BM_UPLOADED_ARCHIVES=${BM_REPOSITORY_ROOT}/${BM_ARCHIVE_PREFIX}-uploaded.list
+
 ##############################################################
 # The SSH method
 #############################################################

diff --git a/lib/actions.sh b/lib/actions.sh
@@ -52,13 +52,12 @@ function make_archives()
     esac
 
     # Now make sure the md5 file is okay.
-    md5file="$BM_REPOSITORY_ROOT/${BM_ARCHIVE_PREFIX}-${TODAY}.md5"
-    if [[ -e $md5file ]] && 
+    if [[ -e $MD5FILE ]] && 
        [[ "$BM_REPOSITORY_SECURE" = "true" ]]; then
-        chown $BM_REPOSITORY_USER:$BM_REPOSITORY_GROUP $md5file ||
-            warning "Unable to change the owner of \"\$md5file\"."
-        chmod $BM_ARCHIVE_CHMOD $md5file ||
-            warning "Unable to change file permissions of \"\$md5file\"."
+        chown $BM_REPOSITORY_USER:$BM_REPOSITORY_GROUP $MD5FILE ||
+            warning "Unable to change the owner of \"\$MD5FILE\"."
+        chmod $BM_ARCHIVE_CHMOD $MD5FILE ||
+            warning "Unable to change file permissions of \"\$MD5FILE\"."
     fi
 done
 }

diff --git a/lib/backup-methods.sh b/lib/backup-methods.sh
@@ -38,15 +38,13 @@ function commit_archive()
         echo "$str ${md5hash})"
     fi
 
-    md5file="$BM_REPOSITORY_ROOT/${BM_ARCHIVE_PREFIX}-${TODAY}.md5"
-
     # Check if the md5file contains already the md5sum of the file_to_create.
     # In this case, the new md5sum overwrites the old one.
-    if grep "$base" $md5file >/dev/null 2>&1 ; then
-        previous_md5sum=$(get_md5sum_from_file $base $md5file)
-        sed -e "/$base/s/$previous_md5sum/$md5hash/" -i $md5file
+    if grep "$base" $MD5FILE >/dev/null 2>&1 ; then
+        previous_md5sum=$(get_md5sum_from_file $base $MD5FILE)
+        sed -e "/$base/s/$previous_md5sum/$md5hash/" -i $MD5FILE
     else
-        echo "$md5hash  $base" >> $md5file
+        echo "$md5hash  $base" >> $MD5FILE
     fi
 
     # Now that the file is created, remove previous duplicates if exists...
@@ -239,7 +237,9 @@ function __get_flags_relative_blacklist()
     target="$2"
     debug "__get_flags_relative_blacklist ($switch, $target)"
 
-    target=${target%/}
+    if [ "$target" != "/" ]; then
+        target=${target%/}
+    fi
     blacklist=""
     for pattern in $BM_TARBALL_BLACKLIST
     do
@@ -253,7 +253,13 @@ function __get_flags_relative_blacklist()
                 # making a relative path...
                 pattern="${pattern#$target}"
                 length=$(expr length $pattern)
-                pattern=$(expr substr $pattern 2 $length)
+                # for $target="/", no spare / is left at the beggining
+                # after the # substitution; thus take substr from pos 1
+                if [ "$target" != "/" ]; then
+                    pattern=$(expr substr $pattern 2 $length)
+                else
+                    pattern=$(expr substr $pattern 1 $length)
+                fi
 
                 # ...and blacklisting it
                 blacklist="$blacklist ${switch}${pattern}"

diff --git a/lib/files.sh b/lib/files.sh
@@ -413,10 +413,9 @@ function purge_duplicate_archives()
                 error "Unable to get date from file."
 
             # get the md5 hash of the file we parse, in its .md5 file
-            md5file="$BM_REPOSITORY_ROOT/${BM_ARCHIVE_PREFIX}-${date_of_file}.md5"
-            md5sum_to_check="$(get_md5sum_from_file $file $md5file)"
+            md5sum_to_check="$(get_md5sum_from_file $file $MD5FILE)"
             if [[ -z "$md5sum_to_check" ]]; then
-                warning "Unable to find the md5 hash of file \"\$file\" in file \"\$md5file\"."
+                warning "Unable to find the md5 hash of file \"\$file\" in file \"\$MD5FILE\"."
                 continue
             fi
 

diff --git a/lib/upload-methods.sh b/lib/upload-methods.sh
@@ -64,7 +64,7 @@ function bm_upload_ssh()
           -h="$bm_upload_hosts" \
           -u="$BM_UPLOAD_SSH_USER" \
           -d="$BM_UPLOAD_SSH_DESTINATION" \
-          -r="$BM_REPOSITORY_ROOT" today 2>$logfile || 
+          -r="$BM_REPOSITORY_ROOT" ${TODAY} 2>$logfile || 
     error "Error reported by backup-manager-upload for method \"scp\", check \"\$logfile\"."
     rm -f $logfile
 }
@@ -101,7 +101,7 @@ function bm_upload_ssh_gpg()
          -u="$BM_UPLOAD_SSH_USER" \
          -d="$BM_UPLOAD_SSH_DESTINATION" \
          -r="$BM_REPOSITORY_ROOT" \
-         --gpg-recipient="$BM_UPLOAD_SSHGPG_RECIPIENT" today 2>$logfile|| 
+         --gpg-recipient="$BM_UPLOAD_SSHGPG_RECIPIENT" ${TODAY} 2>$logfile|| 
     error "Error reported by backup-manager-upload for method \"ssh-gpg\", check \"\$logfile\"."
     rm -f $logfile
 }
@@ -142,7 +142,7 @@ function bm_upload_ftp()
         -h="$bm_upload_hosts" \
         -u="$BM_UPLOAD_FTP_USER" \
         -d="$BM_UPLOAD_FTP_DESTINATION" \
-        -r="$BM_REPOSITORY_ROOT" today 2>$logfile|| 
+        -r="$BM_REPOSITORY_ROOT" ${TODAY} 2>$logfile|| 
     error "Error reported by backup-manager-upload for method \"ftp\", check \"\$logfile\"."
     rm -f $logfile
 
@@ -172,7 +172,7 @@ function bm_upload_s3()
         -h="$bm_upload_hosts" \
         -u="$BM_UPLOAD_S3_ACCESS_KEY" \
         -b="$BM_UPLOAD_S3_DESTINATION" \
-        -r="$BM_REPOSITORY_ROOT" today 2>$logfile || 
+        -r="$BM_REPOSITORY_ROOT" ${TODAY} 2>$logfile || 
     error "Error reported by backup-manager-upload for method \"s3\", check \"\$logfile\"."
     rm -f $logfile
 }