Permalink
Browse files

Move to rsync based backup, both locally and remotely. Local only sav…

…es the last backup.
  • Loading branch information...
perlDreamer committed Dec 13, 2011
1 parent ca28628 commit 0b48f144800b7af390226b3842b952a092c1b3df
Showing with 42 additions and 239 deletions.
  1. +33 −225 wre/sbin/backup.pl
  2. +9 −14 wre/var/setupfiles/wre.conf
@@ -18,6 +18,7 @@
use WRE::Mysql;
use Getopt::Long ();
use Pod::Usage ();
use 5.010;

my $config = WRE::Config->new;
my $util = WRE::File->new(wreConfig => $config);
@@ -32,74 +33,9 @@
# are backups enabled
exit unless $config->get("backup/enabled");

rotateBackupFiles($config);
backupMysql($config);
backupDomains($config);
backupWebgui($config);
backupWre($config);
backupCustom($config);
runExternalScripts($config);
compressBackups($config);
copyToRemote($config);
#removeBackupFiles($config);

#-------------------------------------------------------------------
sub backupCustom {
my $config = shift;

return undef unless $config->get("backup/items/custom");

my $backupDir = dir($config->get("backup/path"));
my $customFile = $config->getWebguiRoot("/sbin/preload.custom");
open FILE, $customFile or die $!;
my @lines = <FILE>;
my @customDirs;
foreach my $line (@lines) {
if ($line =~ /^\//) {
my $backupFile = join("",map { ucfirst($_) } split /\//, $line);
chomp $backupFile;

eval { $util->tar(
file => $backupDir->file($backupFile.".tar")->stringify,
stuff => [$line],
absPath => 1,
)};
print $@."\n" if ($@);
}
else {
next;
}
}
}

#-------------------------------------------------------------------
sub backupDomains {
my $config = shift;

# should we run?
return undef unless $config->get("backup/items/domainsFolder");

my $domainsRoot = dir($config->getDomainRoot);

# get domains to backup
opendir(DIR, $domainsRoot->stringify);
my @domains = readdir(DIR);
closedir(DIR);

# create tarballs
my $tar = $config->get("tar");
my $backupDir = dir($config->get("backup/path"));
my $excludes = $config->getRoot("/etc/backup.exclude");
foreach my $domain (@domains) {
next if ($domain eq "." || $domain eq ".." || $domain eq "demo");
eval {$util->tar(
exclude => $excludes,
file => $backupDir->file($domain.".tar")->stringify,
stuff => [$domainsRoot->subdir($domain)->stringify],
)};
print $@."\n" if ($@);
}
}

#-------------------------------------------------------------------
sub backupMysql {
@@ -143,183 +79,55 @@ sub backupMysql {
}

#-------------------------------------------------------------------
sub backupWebgui {
my $config = shift;

# should we run?
return undef unless $config->get("backup/items/webgui");

# create tarball
eval {$util->tar(
exclude => $config->getRoot("/etc/backup.exclude"),
file => file($config->get("backup/path"), "webgui.tar")->stringify,
stuff => [$config->getWebguiRoot],
)};
print $@."\n" if ($@);
}

#-------------------------------------------------------------------
sub backupWre {
my $config = shift;
my $full = $config->get("backup/items/fullWre");
my $small = $config->get("backup/items/smallWre");

# should we run?
return undef unless ($full || $small);

# whole thing or just configs?
my $pathToBackup = ($full) ? $config->getRoot : $config->getRoot("/etc");

# create tarball
eval {$util->tar(
exclude => $config->getRoot("/etc/backup.exclude"),
file => file($config->get("backup/path"), "wre.tar")->stringify,
stuff => [$pathToBackup],
)};
print $@."\n" if ($@);
}

#-------------------------------------------------------------------
sub compressBackups {
sub backupFiles {
my $config = shift;
my $gzip = file($config->get("gzip"))->stringify;
my $backupDir = dir($config->get("backup/path"));

# compress files
system("$gzip -f ".$backupDir->file("*.sql")->stringify);
system("$gzip -f ".$backupDir->file("*.tar")->stringify);
my $paths = $config->get("backup/items");
my $backupDir = $config->get("backup/path");
foreach my $path (@{ $paths }) {
say "rsyncing $path locally...";
system ("nice rsync -av --exclude=logs --exclude=mysqldata $path $backupDir/backup");
}
}

#-------------------------------------------------------------------
sub copyToRemote {
my $config = shift;
# should we run?
return undef unless $config->get("backup/ftp/enabled");
my $protocol = $config->get("backup/ftp/protocol") || "ftp";

my $now = time;
my $passive = $config->get("backup/ftp/usePassiveTransfers");
my $host = $config->get("backup/ftp/hostname");
my $path = $config->get("backup/ftp/path");
my $user = $config->get("backup/ftp/user");
my $pass = $config->get("backup/ftp/password");
my $rotations = $config->get("backup/ftp/rotations");
my $extraCommands = '';
# don't validate local cert
if ($protocol eq 'https') {
$extraCommands .= 'set ssl:verify-certificate off; ';
}
return undef unless $config->get("backup/rsync/enabled");

# a little massage
$path = ($path eq "/") ? "." : $path; # never let it look at the root of the server
my $user = $config->get("backup/rsync/user");
my $host = $config->get("backup/rsync/hostname");
my $ACCOUNT = $user . '@' . $host;
my $rotations = $config->get("backup/rsync/rotations");
my $remote_path = $config->get("backup/rsync/remote_path");
my $paths = $config->get("backup/items");
my $backupDir = $config->get("backup/path");

# get old versions
if ($rotations > 1) {
my $cmd = $config->getRoot('/prereqs/bin/lftp').' -e "cd '.$path.'; ls; exit" -u '.$user.','.$pass.' '.$protocol.'://'.$host.'/';
my @dirs = ();
if (open my $pipe, $cmd.'|') {
while (my $line = <$pipe>) {
chomp $line;
if ($line =~ m/^([drxws-]+)\s+\d+\s+\w+\s+\w+\s+\d+\s+\w+\s+\d+\s+\d+:\d{2}\s+(\w+)/ || $line =~ m/^([drxws-]+)\s+--\s+(\w+)/) {
my ($privs, $name) = ($1, $2);
# skip non-backup directories
next unless ($privs =~ m/^d/);
next unless ($name =~ m/^\d+$/);
push @dirs, $name;
}
}
close $pipe;
say "Removing last remote backup files...";
my $cmd = "ssh $ACCOUNT rm -rf $remote_path/backup.$rotations";
system($cmd);

# rotate backups except for the last one
for (my $i=$rotations; $i > 1; $i--) {
my $prev= $i - 1;
say "Rotating old remote backup $i...";
$cmd = "ssh $ACCOUNT mv $remote_path/backup.$prev $remote_path/backup.$i";
system($cmd);
}
else {
die "Couldn't connect to backup server.";
}

# do rotations
@dirs = sort(@dirs);
my $i = scalar(@dirs);
foreach my $dir (@dirs) {
last if ($i < $rotations);
system($config->getRoot('/prereqs/bin/lftp').' -e "rm -r -f '.$path.'/'.$dir.'; exit" -u '.$user.','.$pass.' '.$protocol.'://'.$host);
$i--;
}
}

# deal with passive transfers
if ($protocol eq 'ftp' && !$passive) {
$extraCommands .= "set ftp:passive-mode off; ";
}

# don't do the rotations folder if we aren't using rotations
if ($rotations > 1) {
$extraCommands .= 'mkdir '.$path.'/'.$now.'; mput -O '.$path.'/'.$now;
}
else {
$extraCommands .= 'mput -O '.$path;
say "Copying first backup to 1...";
$cmd = "ssh $ACCOUNT cp -al $remote_path/backup $remote_path/backup.1";
system($cmd);
}

# do transfer
my $cmd = $config->getRoot('/prereqs/bin/lftp').' -e "'.$extraCommands.' '.file($config->get("backup/path"),'/*.gz')->stringify.'; exit" -u '.$user.','.$pass.' '.$protocol.'://'.$host;
system($cmd);
}

#-------------------------------------------------------------------
sub removeBackupFiles {
my $config = shift;
my $backupDir = dir($config->get("backup/path"));
my $rotations = $config->get("backup/rotations");
if ($rotations == 0 ||$rotations == 1) {
opendir(DIR,$backupDir->stringify);
my @files = readdir(DIR);
closedir(DIR);
foreach my $file (@files) {
if ($rotations eq "0") {
$backupDir->file($file)->remove;
}
elsif ($file =~ /(.*\.)1/ ) {
$backupDir->file($file)->remove;
}
}
say "Moving new data over...";
foreach my $path (@{ $paths }) {
say "rsyncing $path remotely...";
system ("rsync -av --chmod=u+rwx --exclude=logs --exclude=mysqldata $path $ACCOUNT:$remote_path/backup");
}
}
#-------------------------------------------------------------------
sub rotateBackupFiles {
my $config = shift;
my $backupDir = dir($config->get("backup/path"));
my $rotations = $config->get("backup/rotations") - 1; # .gz counts as 1

# get the list of files
opendir(DIR,$backupDir->stringify);
my @files = readdir(DIR);
closedir(DIR);

# rotate and delete old files
for (my $i = $rotations; $i > 0; $i--) {
foreach my $file (@files) {

# only look at files where their extension is a specific digit
if ($file =~ /(.*\.)$i$/) {
my $filefront = $1;

# get rid of oldest files
if ($i == $rotations) {
$backupDir->file($file)->remove;
}

# rotate younger files
else {
rename $backupDir->file($file)->stringify, $backupDir->file($filefront.($i+1))->stringify;
}
}
}
}

# rotate new files
foreach my $file (@files) {
if ($file =~ /\.gz$/) {
rename $backupDir->file($file)->stringify, $backupDir->file($file.".1")->stringify;
}
}
}

#-------------------------------------------------------------------
sub runExternalScripts {
@@ -85,27 +85,22 @@
"backup" : {
"enabled" : 0,
"path" : "/backup",
"rotations" : 7,
"mysql" : {
"user" : "backup",
"password" : "xxxxxxx"
},
"items" : {
"mysql" : 1,
"fullWre" : 0,
"smallWre" : 1,
"domainsFolder" : 1,
"webgui" : 1
},
"items" : [ ##List of directories, wildcards are okay
"/data/domains", ##All domain files
"/data/WebGUI", ##All of WebGUI, including etc and sbin
"/data/backup/*.sql.gz", ##All database dumps
"/data/wre/etc" ##All WRE configuration files
],
"externalScripts" : [],
"ftp" : {
"rsync" : {
"enabled" : 0,
"usePassiveTransfers" : 1,
"rotations" : 3,
"hostname" : "ftp.example.com",
"user" : "ftpuser",
"protocol" : "ftp",
"password" : "xxxxxx",
"hostname" : "rsync.example.com",
"user" : "rsyncuser",
"path" : "."
}
},

0 comments on commit 0b48f14

Please sign in to comment.