Skip to content

Commit

Permalink
Merge ec55ac6 into 95c2bd8
Browse files Browse the repository at this point in the history
  • Loading branch information
@jimklimov
jimklimov committed Mar 12, 2024
2 parents 95c2bd8 + ec55ac6 commit d90a0c4
Show file tree
Hide file tree
Showing 7 changed files with 337 additions and 14 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/spelling/expect.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ attr
austingroupbugs
autocommit
autoconf
autocreation
autom
automake
automounting
Expand Down Expand Up @@ -71,6 +72,7 @@ Bznapzendzetup
Bznapzendztatz
canmount
CBuilder
cdn
cfg
cgi
cgit
Expand Down Expand Up @@ -377,6 +379,7 @@ newestbe
nf
nh
noaction
noauto
nobase
nodelay
nodestroy
Expand Down
2 changes: 2 additions & 0 deletions CHANGES
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ znapzend (0.21.3) unstable; urgency=medium
* Extended handling of "org.znapzend:enabled=off" setting for sub-trees: now if the same intermediate dataset declares "org.znapzend:recursive=on", the disablement of znapzend handling takes place for descendants as well (previously it had effect only exactly for datasets that set "org.znapzend:enabled=off" as a local ZFS property)
* Fixed CI recipes and contents for spell-checker
* Added rc-script and integration documentation for FreeBSD and similar platforms
* Extended `--autoCreation` effect (or lack thereof) to newly appearing sub-datasets; added a `--noautoCreation` option to help override configuration file settings (where used)
* Introduced `dst_N_autocreation` setting via ZFS properties (per-destination, inheritable)

-- Jim Klimov <jimklimov@gmail.com> Tue, 9 Jan 2024 13:42:28 +0100

Expand Down
22 changes: 21 additions & 1 deletion bin/znapzend
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ sub main {
my $opts = {};
GetOptions($opts, qw(help|h man debug|d noaction|n nodestroy features=s),
qw(sudo pfexec rootExec=s daemonize pidfile=s logto=s loglevel=s),
qw(runonce:s connectTimeout=s timeWarp=i nodelay autoCreation version),
qw(runonce:s connectTimeout=s timeWarp=i nodelay autoCreation noautoCreation version),
qw(forcedSnapshotSuffix=s forbidDestRollback),
qw(skipIntermediates|i sendIntermediates|I),
# Note: the intended usage is either via feature as done
Expand Down Expand Up @@ -85,6 +85,18 @@ sub main {
$opts->{forbidDestRollback} = 0;
}

# Note: default is "undef" to use a ZFS property dst_N_autocreation
# (lower-case "c" in the name) if present; finally assumes 0 (false)
# if not set in any configuration source for a particular dataset.
if (defined($opts->{noautoCreation})) {
$opts->{autoCreation} = 0;
delete $opts->{noautoCreation};
} else {
if (defined($opts->{autoCreation})) {
$opts->{autoCreation} = 1;
}
}

if (defined($opts->{sinceForced}) && ($opts->{sinceForced} eq '')) {
delete $opts->{sinceForced};
}
Expand Down Expand Up @@ -202,6 +214,8 @@ B<znapzend> [I<options>...]
--connectTimeout=x sets the ConnectTimeout for ssh commands
--autoCreation automatically create dataset on destination if it does
not exist
--noautoCreation avoid automatically creating a dataset on destination
if it does not exist (default)
--timeWarp=x shift znapzend's sense of NOW into the future
by x seconds
--skipOnPreSnapCmdFail skip snapshots if the pre-snap-command fails
Expand Down Expand Up @@ -610,6 +624,12 @@ sets the ssh connection timeout (in seconds)
Automatically create a dataset on a destination host if it's not there yet.
=item B<--noautoCreation>
Avoid automatically creating a dataset on a destination host if it's not
there yet. This is the default behavior; the option is available to help
explicitly override a setting inherited from a configuration file, etc.
=item B<--timeWarp>=x
Shift ZnapZend's sense of time into the future by x seconds.
Expand Down
54 changes: 54 additions & 0 deletions bin/znapzendzetup
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,54 @@ sub main {

last;
};
/^enable-dst-autocreation$/ && do {
$opts->{dst} = pop @ARGV;
$opts->{src} = pop @ARGV;
if (!defined $opts->{src}) {
pod2usage(-exitval => 'NOEXIT');
die ("ERROR: source argument for option $mainOpt was not provided\n");
}
if (!defined $opts->{dst}) {
pod2usage(-exitval => 'NOEXIT');
die ("ERROR: destination argument for option $mainOpt was not provided\n");
}
$zConfig->enableBackupSetDstAutoCreation($opts->{src}, $opts->{dst})
or die "ERROR: cannot enable backup config for $opts->{src} destination $opts->{dst}. Did you create this config?\n";

last;
};
/^disable-dst-autocreation$/ && do {
$opts->{dst} = pop @ARGV;
$opts->{src} = pop @ARGV;
if (!defined $opts->{src}) {
pod2usage(-exitval => 'NOEXIT');
die ("ERROR: source argument for option $mainOpt was not provided\n");
}
if (!defined $opts->{dst}) {
pod2usage(-exitval => 'NOEXIT');
die ("ERROR: destination argument for option $mainOpt was not provided\n");
}
$zConfig->disableBackupSetDstAutoCreation($opts->{src}, $opts->{dst})
or die "ERROR: cannot disable backup config for $opts->{src} destination $opts->{dst}. Did you create this config?\n";

last;
};
/^inherit-dst-autocreation$/ && do {
$opts->{dst} = pop @ARGV;
$opts->{src} = pop @ARGV;
if (!defined $opts->{src}) {
pod2usage(-exitval => 'NOEXIT');
die ("ERROR: source argument for option $mainOpt was not provided\n");
}
if (!defined $opts->{dst}) {
pod2usage(-exitval => 'NOEXIT');
die ("ERROR: destination argument for option $mainOpt was not provided\n");
}
$zConfig->inheritBackupSetDstAutoCreation($opts->{src}, $opts->{dst})
or die "ERROR: cannot disable backup config for $opts->{src} destination $opts->{dst}. Did you create this config?\n";

last;
};
/^list$/ && do {
GetOptions($opts, (@ROOT_EXEC_OPTIONS, qw(recursive|r inherited))) or exit 1;

Expand Down Expand Up @@ -591,6 +639,12 @@ and where 'command' and its unique options is one of the following:
disable-dst <src_dataset> <DST_key>
enable-dst-autocreation <src_dataset> <DST_key>
disable-dst-autocreation <src_dataset> <DST_key>
inherit-dst-autocreation <src_dataset> <DST_key>
list [--recursive] [--inherited] [src_dataset...]
export <src_dataset>
Expand Down
101 changes: 90 additions & 11 deletions lib/ZnapZend.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ has pidfile => sub { q{} };
has forcedSnapshotSuffix => sub { q{} };
has defaultPidFile => sub { q{/var/run/znapzend.pid} };
has terminate => sub { 0 };
has autoCreation => sub { 0 };
has autoCreation => sub { undef };
has timeWarp => sub { undef };
has nodelay => sub { 0 };
has skipOnPreSnapCmdFail => sub { 0 };
Expand Down Expand Up @@ -404,14 +404,23 @@ my $refreshBackupPlans = sub {
#create backup hashes for all destinations
for (keys %$backupSet){
my ($key) = /^dst_([^_]+)_plan$/ or next;
my $autoCreation = $self->autoCreation;
if (!defined($autoCreation)) {
# Caller did not require any particular behavior, so
# check the ZFS property name (note lower-case "c"):
$autoCreation = (exists $backupSet->{"dst_$key" . '_autocreation'} ? $backupSet->{"dst_$key" . '_autocreation'} : undef);
}
if (!defined($autoCreation)) {
$autoCreation = 0;
}

#check if destination exists (i.e. is valid) otherwise recheck as dst might be online, now
if (!$backupSet->{"dst_$key" . '_valid'}){

$backupSet->{"dst_$key" . '_valid'} =
$self->zZfs->dataSetExists($backupSet->{"dst_$key"}) or do {

if ($self->autoCreation && !$self->sendRaw) {
if ($autoCreation && !$self->sendRaw) {
my ($zpool) = $backupSet->{"dst_$key"} =~ /(^[^\/]+)\//;

# check if we can access destination zpool, if so create parent dataset
Expand All @@ -429,8 +438,10 @@ my $refreshBackupPlans = sub {
$backupSet->{"dst_$key" . '_valid'} or
$self->zLog->warn("destination '" . $backupSet->{"dst_$key"}
. "' does not exist or is offline. will be rechecked every run..."
. ( $self->autoCreation ? "" : " Consider running znapzend --autoCreation" ) );
. ( $autoCreation ? "" : " Consider running znapzend --autoCreation" ) );
};

$self->zLog->debug('refreshBackupPlans(): detected dst_' . $key . '_valid status for ' . $backupSet->{"dst_$key"} . ': ' . $backupSet->{"dst_$key" . '_valid'}) if ($self->debug);
}
$backupSet->{"dst$key" . 'PlanHash'}
= $self->zTime->backupPlanToHash($backupSet->{"dst_$key" . '_plan'});
Expand Down Expand Up @@ -561,13 +572,29 @@ my $sendRecvCleanup = sub {
#recheck non valid dst as it might be online, now
if (!$backupSet->{"dst_$key" . '_valid'}) {

my $autoCreation = $self->autoCreation;
if (!defined($autoCreation)) {
# Caller did not require any particular behavior, so
# check the ZFS property name (note lower-case "c").
# Note we are looking at "root" datasets with a backup
# schedule here (enumerated earlier); children if any
# would be checked below:
$autoCreation = (exists $backupSet->{"dst_$key" . '_autocreation'} ? $backupSet->{"dst_$key" . '_autocreation'} : undef);
}
if (!defined($autoCreation)) {
$autoCreation = 0;
}

$backupSet->{"dst_$key" . '_valid'} =
$self->zZfs->dataSetExists($backupSet->{"dst_$key"}) or do {

if ($self->autoCreation && !$self->sendRaw) {
if ($autoCreation && !$self->sendRaw) {
my ($zpool) = $backupSet->{"dst_$key"} =~ /(^[^\/]+)\//;

# check if we can access destination zpool, if so create parent dataset
# check if we can access destination zpool, if so -
# create parent dataset (e.g. this backupSet root;
# note that if we recurse into children that may be
# absent, they are treated separately below)
$self->zZfs->dataSetExists($zpool) && do {
$self->zLog->info("creating destination dataset '" . $backupSet->{"dst_$key"} . "'...");

Expand All @@ -579,15 +606,23 @@ my $sendRecvCleanup = sub {
}
};
}
( $backupSet->{"dst_$key" . '_valid'} || ($self->sendRaw && $self->autoCreation) ) or do {
( $backupSet->{"dst_$key" . '_valid'} || ($self->sendRaw && $autoCreation) ) or do {
my $errmsg = "destination '" . $backupSet->{"dst_$key"}
. "' does not exist or is offline. ignoring it for this round...";
$self->zLog->warn($errmsg);
push (@sendFailed, $errmsg);
$thisSendFailed = 1;
. "' does not exist or is offline; ignoring it for this round...";
# Avoid spamming for every loop cycle, if we do not have
# the dataset and know we do not intend to auto-create it
$self->zLog->warn($errmsg) if ($autoCreation or $self->debug);
if (!$autoCreation) {
$self->zLog->warn("Autocreation is disabled for this dataset or whole run, so skipping without error") if ($self->debug);
} else {
push (@sendFailed, $errmsg);
$thisSendFailed = 1;
}
next;
};
};

$self->zLog->debug('sendRecvCleanup(): detected dst_' . $key . '_valid status for ' . $backupSet->{"dst_$key"} . ': ' . $backupSet->{"dst_$key" . '_valid'}) if ($self->debug);
}

#sending loop through all subdatasets
Expand All @@ -600,17 +635,61 @@ my $sendRecvCleanup = sub {
my $dstDataSet = $srcDataSet;
$dstDataSet =~ s/^\Q$backupSet->{src}\E/$backupSet->{$dst}/;

my $autoCreation = $self->autoCreation;
if (!defined($autoCreation)) {
# Caller did not require any particular behavior, so
# check the ZFS property name (note lower-case "c").
# Look at properties of this dataset, allow inherited
# values. TOTHINK: Get properties once for all tree?
my $properties = $self->zZfs->getDataSetProperties($srcDataSet, 0, 1);
if ($properties->[0]) {
for my $prop (keys %{$properties->[0]}) {
if ($prop eq "dst_$key" . '_autocreation') {
$autoCreation = (%{$properties->[0]}{$prop} eq "on" ? 1 : 0);
last;
}
}
}
}
if (!defined($autoCreation)) {
$autoCreation = 0;
}

$self->zLog->debug('sending snapshots from ' . $srcDataSet . ' to ' . $dstDataSet . ((grep (/^\Q$srcDataSet\E$/, @dataSetsExplicitlyDisabled)) ? ": not enabled, should be skipped" : ""));
{

# Time to check if the target sub-dataset exists
# at all (unless we would auto-create one anyway).
if ((!$autoCreation || !$self->sendRaw) && !($self->zZfs->dataSetExists($dstDataSet))) {
my $errmsg = "sub-destination '" . $dstDataSet
. "' does not exist or is offline; ignoring it for this round... Consider "
. ( $autoCreation || $self->sendRaw ? "" : "running znapzend --autoCreation or " )
. "disabling this dataset from znapzend handling.";
# Avoid spamming for every loop cycle, if we do not have
# the dataset and know we do not intend to auto-create it
$self->zLog->warn($errmsg) if ($autoCreation or $self->debug);
if (!$autoCreation) {
$self->zLog->warn("Autocreation is disabled for this dataset or whole run, so skipping without error") if ($self->debug);
} else {
push (@sendFailed, $errmsg);
$thisSendFailed = 1;
}
next;
}

{ # scoping
local $@;
eval {
local $SIG{__DIE__};
$self->zLog->debug('Are we sending "--since"? '.
'since=="' . $self->since . '"'.
', skipIntermediates=="' . $self->skipIntermediates . '"' .
', forbidDestRollback=="' . $self->forbidDestRollback . '"' .
', autoCreation=="' . ( $autoCreation ? "true" : "false" ) . '"' .
', sendRaw=="' . $self->sendRaw . '"' .
', valid=="' . ( $backupSet->{"dst_$key" . '_valid'} ? "true" : "false" ) . '"' .
', justCreated=="' . ( $backupSet->{"dst_$key" . '_justCreated'} ? "true" : "false" ) . '"'
) if $self->debug;

if ($self->since) {
# Make sure that if we use the "--sinceForced=X" or
# "--since=X" option, this named snapshot exists (or
Expand Down

0 comments on commit d90a0c4

Please sign in to comment.