Skip to content

Commit

Permalink
Merge branch 'bookmarks' into force-delete
Browse files Browse the repository at this point in the history
  • Loading branch information
@phreaker0
phreaker0 committed Aug 7, 2018
2 parents 020f57c + 7070d81 commit 16613b6
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 21 deletions.
26 changes: 26 additions & 0 deletions README.md
View file
Expand Up @@ -118,6 +118,32 @@ If ZFS supports resumeable send/receive streams on both the source and target th

As of 1.4.18, syncoid also automatically supports and enables resume of interrupted replication when both source and target support this feature.

##### Syncoid Dataset Properties

+ syncoid:sync

Available values:

+ `true` (default if unset)

This dataset will be synchronised to all hosts.

+ `false`

This dataset will not be synchronised to any hosts - it will be skipped. This can be useful for preventing certain datasets from being transferred when recursively handling a tree.

+ `host1,host2,...`

A comma separated list of hosts. This dataset will only be synchronised by hosts listed in the property.

_Note_: this check is performed by the host running `syncoid`, thus the local hostname must be present for inclusion during a push operation // the remote hostname must be present for a pull.

_Note_: this will also prevent syncoid from handling the dataset if given explicitly on the command line.

_Note_: syncing a child of a no-sync dataset will currently result in a critical error.

_Note_: empty properties will be handled as if they were unset.

##### Syncoid Command Line Options

+ [source]
Expand Down
98 changes: 77 additions & 21 deletions syncoid
View file
Expand Up @@ -98,6 +98,7 @@ my $targetsudocmd = $targetisroot ? '' : $sudocmd;
my %avail = checkcommands();

my %snaps;
my $exitcode = 0;

## break here to call replication individually so that we ##
## can loop across children separately, for recursive ##
Expand Down Expand Up @@ -128,7 +129,7 @@ if ($targethost ne '') {
close FH;
}

exit 0;
exit $exitcode;

##############################################################################
##############################################################################
Expand Down Expand Up @@ -184,9 +185,27 @@ sub syncdataset {

if ($debug) { print "DEBUG: syncing source $sourcefs to target $targetfs.\n"; }

my $sync = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'syncoid:sync');

if ($sync eq 'true' || $sync eq '-' || $sync eq '') {
# empty is handled the same as unset (aka: '-')
# definitely sync this dataset - if a host is called 'true' or '-', then you're special
} elsif ($sync eq 'false') {
if (!$quiet) { print "INFO: Skipping dataset (syncoid:sync=false): $sourcefs...\n"; }
return 0;
} else {
my $hostid = hostname();
my @hosts = split(/,/,$sync);
if (!(grep $hostid eq $_, @hosts)) {
if (!$quiet) { print "INFO: Skipping dataset (syncoid:sync doesn't include $hostid): $sourcefs...\n"; }
return 0;
}
}

# make sure target is not currently in receive.
if (iszfsbusy($targethost,$targetfs,$targetisroot)) {
warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n";
if ($exitcode < 1) { $exitcode = 1; }
return 0;
}

Expand Down Expand Up @@ -232,11 +251,16 @@ sub syncdataset {
if (!defined $args{'no-sync-snap'} && !defined $skipsnapshot) {
# create a new syncoid snapshot on the source filesystem.
$newsyncsnap = newsyncsnap($sourcehost,$sourcefs,$sourceisroot);
if (!$newsyncsnap) {
# we already whined about the error
return 0;
}
} else {
# we don't want sync snapshots created, so use the newest snapshot we can find.
$newsyncsnap = getnewestsnapshot($sourcehost,$sourcefs,$sourceisroot);
if ($newsyncsnap eq 0) {
warn "CRITICAL: no snapshots exist on source $sourcefs, and you asked for --no-sync-snap.\n";
if ($exitcode < 1) { $exitcode = 1; }
return 0;
}
}
Expand Down Expand Up @@ -265,6 +289,11 @@ sub syncdataset {
}
my $oldestsnap = getoldestsnapshot(\%snaps);
if (! $oldestsnap) {
if (defined ($args{'no-sync-snap'}) ) {
# we already whined about the missing snapshots
return 0;
}

# getoldestsnapshot() returned false, so use new sync snapshot
if ($debug) { print "DEBUG: getoldestsnapshot() returned false, so using $newsyncsnap.\n"; }
$oldestsnap = $newsyncsnap;
Expand Down Expand Up @@ -293,10 +322,14 @@ sub syncdataset {
# make sure target is (still) not currently in receive.
if (iszfsbusy($targethost,$targetfs,$targetisroot)) {
warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n";
if ($exitcode < 1) { $exitcode = 1; }
return 0;
}
system($synccmd) == 0
or die "CRITICAL ERROR: $synccmd failed: $?";
system($synccmd) == 0 or do {
warn "CRITICAL ERROR: $synccmd failed: $?";
if ($exitcode < 2) { $exitcode = 2; }
return 0;
};

# now do an -I to the new sync snapshot, assuming there were any snapshots
# other than the new sync snapshot to begin with, of course - and that we
Expand All @@ -319,16 +352,20 @@ sub syncdataset {
# make sure target is (still) not currently in receive.
if (iszfsbusy($targethost,$targetfs,$targetisroot)) {
warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n";
if ($exitcode < 1) { $exitcode = 1; }
return 0;
}

if (!$quiet) { print "INFO: Updating new target filesystem with incremental $sourcefs\@$oldestsnap ... $newsyncsnap (~ $disp_pvsize):\n"; }
if ($debug) { print "DEBUG: $synccmd\n"; }

if ($oldestsnap ne $newsyncsnap) {
system($synccmd) == 0
or warn "CRITICAL ERROR: $synccmd failed: $?";
my $ret = system($synccmd);
if ($ret != 0) {
warn "CRITICAL ERROR: $synccmd failed: $?";
if ($exitcode < 1) { $exitcode = 1; }
return 0;
}
} else {
if (!$quiet) { print "INFO: no incremental sync needed; $oldestsnap is already the newest available snapshot.\n"; }
}
Expand All @@ -352,8 +389,11 @@ sub syncdataset {

if (!$quiet) { print "Resuming interrupted zfs send/receive from $sourcefs to $targetfs (~ $disp_pvsize remaining):\n"; }
if ($debug) { print "DEBUG: $synccmd\n"; }
system("$synccmd") == 0
or die "CRITICAL ERROR: $synccmd failed: $?";
system("$synccmd") == 0 or do {
warn "CRITICAL ERROR: $synccmd failed: $?";
if ($exitcode < 2) { $exitcode = 2; }
return 0;
};

# a resumed transfer will only be done to the next snapshot,
# so do an normal sync cycle
Expand Down Expand Up @@ -418,6 +458,7 @@ sub syncdataset {
}

# if we got this far, we failed to find a matching snapshot/bookmark.
if ($exitcode < 2) { $exitcode = 2; }

print "\n";
print "CRITICAL ERROR: Target $targetfs exists but has no snapshots matching with $sourcefs!\n";
Expand All @@ -442,6 +483,7 @@ sub syncdataset {
# make sure target is (still) not currently in receive.
if (iszfsbusy($targethost,$targetfs,$targetisroot)) {
warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n";
if ($exitcode < 1) { $exitcode = 1; }
return 0;
}

Expand Down Expand Up @@ -490,8 +532,11 @@ sub syncdataset {

if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $nextsnapshot (~ $disp_pvsize):\n"; }
if ($debug) { print "DEBUG: $synccmd\n"; }
system("$synccmd") == 0
or die "CRITICAL ERROR: $synccmd failed: $?";
system("$synccmd") == 0 or do {
warn "CRITICAL ERROR: $synccmd failed: $?";
if ($exitcode < 2) { $exitcode = 2; }
return 0;
};

$matchingsnap = $nextsnapshot;
$matchingsnapescaped = escapeshellparam($matchingsnap);
Expand All @@ -502,8 +547,11 @@ sub syncdataset {

if (!$quiet) { print "Sending incremental $sourcefs#$bookmarkescaped ... $newsyncsnap (~ $disp_pvsize):\n"; }
if ($debug) { print "DEBUG: $synccmd\n"; }
system("$synccmd") == 0
or die "CRITICAL ERROR: $synccmd failed: $?";
system("$synccmd") == 0 or do {
warn "CRITICAL ERROR: $synccmd failed: $?";
if ($exitcode < 2) { $exitcode = 2; }
return 0;
};
}
}

Expand All @@ -519,8 +567,11 @@ sub syncdataset {

if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; }
if ($debug) { print "DEBUG: $synccmd\n"; }
system("$synccmd") == 0
or die "CRITICAL ERROR: $synccmd failed: $?";
system("$synccmd") == 0 or do {
warn "CRITICAL ERROR: $synccmd failed: $?";
if ($exitcode < 2) { $exitcode = 2; }
return 0;
};
}

# restore original readonly value to target after sync complete
Expand Down Expand Up @@ -691,7 +742,7 @@ sub checkcommands {
if ($debug) { print "DEBUG: checking availability of $mbuffercmd on source...\n"; }
$avail{'sourcembuffer'} = `$sourcessh $lscmd $mbuffercmd 2>/dev/null`;
if ($avail{'sourcembuffer'} eq '') {
print "WARN: $mbuffercmd not available on source $s - sync will continue without source buffering.\n";
if (!$quiet) { print "WARN: $mbuffercmd not available on source $s - sync will continue without source buffering.\n"; }
$avail{'sourcembuffer'} = 0;
} else {
$avail{'sourcembuffer'} = 1;
Expand All @@ -700,7 +751,7 @@ sub checkcommands {
if ($debug) { print "DEBUG: checking availability of $mbuffercmd on target...\n"; }
$avail{'targetmbuffer'} = `$targetssh $lscmd $mbuffercmd 2>/dev/null`;
if ($avail{'targetmbuffer'} eq '') {
print "WARN: $mbuffercmd not available on target $t - sync will continue without target buffering.\n";
if (!$quiet) { print "WARN: $mbuffercmd not available on target $t - sync will continue without target buffering.\n"; }
$avail{'targetmbuffer'} = 0;
} else {
$avail{'targetmbuffer'} = 1;
Expand All @@ -712,14 +763,14 @@ sub checkcommands {
$avail{'localmbuffer'} = `$lscmd $mbuffercmd 2>/dev/null`;
if ($avail{'localmbuffer'} eq '') {
$avail{'localmbuffer'} = 0;
print "WARN: $mbuffercmd not available on local machine - sync will continue without local buffering.\n";
if (!$quiet) { print "WARN: $mbuffercmd not available on local machine - sync will continue without local buffering.\n"; }
}
}

if ($debug) { print "DEBUG: checking availability of $pvcmd on local machine...\n"; }
$avail{'localpv'} = `$lscmd $pvcmd 2>/dev/null`;
if ($avail{'localpv'} eq '') {
print "WARN: $pvcmd not available on local machine - sync will continue without progress bar.\n";
if (!$quiet) { print "WARN: $pvcmd not available on local machine - sync will continue without progress bar.\n"; }
$avail{'localpv'} = 0;
} else {
$avail{'localpv'} = 1;
Expand Down Expand Up @@ -847,7 +898,7 @@ sub getoldestsnapshot {
# must not have had any snapshots on source - luckily, we already made one, amirite?
if (defined ($args{'no-sync-snap'}) ) {
# well, actually we set --no-sync-snap, so no we *didn't* already make one. Whoops.
die "CRIT: --no-sync-snap is set, and getoldestsnapshot() could not find any snapshots on source!\n";
warn "CRIT: --no-sync-snap is set, and getoldestsnapshot() could not find any snapshots on source!\n";
}
return 0;
}
Expand All @@ -856,7 +907,7 @@ sub getnewestsnapshot {
my $snaps = shift;
foreach my $snap ( sort { $snaps{'source'}{$b}{'creation'}<=>$snaps{'source'}{$a}{'creation'} } keys %{ $snaps{'source'} }) {
# return on first snap found - it's the newest
print "NEWEST SNAPSHOT: $snap\n";
if (!$quiet) { print "NEWEST SNAPSHOT: $snap\n"; }
return $snap;
}
# must not have had any snapshots on source - looks like we'd better create one!
Expand All @@ -869,6 +920,7 @@ sub getnewestsnapshot {
# we also probably need an argument to mute this WARN, for people who deliberately exclude
# datasets from recursive replication this way.
warn "WARN: --no-sync-snap is set, and getnewestsnapshot() could not find any snapshots on source for current dataset. Continuing.\n";
if ($exitcode < 2) { $exitcode = 2; }
}
return 0;
}
Expand Down Expand Up @@ -1039,8 +1091,12 @@ sub newsyncsnap {
my %date = getdate();
my $snapname = "syncoid\_$identifier$hostid\_$date{'stamp'}";
my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fsescaped\@$snapname\n";
system($snapcmd) == 0
or die "CRITICAL ERROR: $snapcmd failed: $?";
system($snapcmd) == 0 or do {
warn "CRITICAL ERROR: $snapcmd failed: $?";
if ($exitcode < 2) { $exitcode = 2; }
return 0;
};

return $snapname;
}

Expand Down

0 comments on commit 16613b6

Please sign in to comment.