Use bookmarks created after the latest snapshot

This commit changes syncoid's behavior so it is always looking for
a matching snapshot and a matching bookmark. If the bookmark was created
after the snapshot it is used instead. This allows replication when the
latest snapshot replicated was deleted on the source, a common snapshot
was found but rollback on the target is not allowed. The matching bookmark
is used instead for replication.

This fixes jimsalterjrs#602

syncoid

@@ -410,6 +410,7 @@ sub syncdataset {
 
 	my $newsyncsnap;
 	my $matchingsnap;
+	my $usebookmark = 0;
 
 	# skip snapshot checking/creation in case of resumed receive
 	if (!defined($receivetoken)) {
@@ -585,24 +586,27 @@ sub syncdataset {
 		my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used');
 
 		my %bookmark = ();
+		my $bookmarksnap = 0;
 
 		$matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps);
-		if (! $matchingsnap) {
-			# no matching snapshots, check for bookmarks as fallback
-			my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot);
 
-			# check for matching guid of source bookmark and target snapshot (oldest first)
-			foreach my $snap ( sort { sortsnapshots(\%snaps, $b, $a, 'target') } keys %{ $snaps{'target'} }) {
-				my $guid = $snaps{'target'}{$snap}{'guid'};
+		# find most recent matching bookmark
+		my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot);
 
-				if (defined $bookmarks{$guid}) {
-					# found a match
-					%bookmark = %{ $bookmarks{$guid} };
-					$matchingsnap = $snap;
-					last;
-				}
+		# check for matching guid of source bookmark and target snapshot (oldest first)
+		foreach my $snap ( sort { sortsnapshots(\%snaps, $b, $a, 'target') } keys %{ $snaps{'target'} }) {
+			my $guid = $snaps{'target'}{$snap}{'guid'};
+
+			if (defined $bookmarks{$guid}) {
+				# found a match
+				%bookmark = %{ $bookmarks{$guid} };
+				$bookmarksnap = $snap;
+				last;
 			}
+		}
 
+		if (! $matchingsnap) {
+			# no matching snapshots, check for bookmarks as fallback
 			if (! %bookmark) {
 				# force delete is not possible for the root dataset
 				if ($args{'force-delete'} && index($targetfs, '/') != -1) {
@@ -657,6 +661,18 @@ sub syncdataset {
 
 				# return false now in case more child datasets need replication.
 				return 0;
+			} else {
+				# use bookmark as fallback
+				$matchingsnap = $bookmarksnap;
+				$usebookmark = 1;
+			}
+		} elsif (%bookmark && $bookmark{'guid'} ne $snaps{'source'}{$matchingsnap}{'guid'}) {
+			my $comparetxg = defined $snaps{'source'}{$matchingsnap}{'createtxg'} && defined $bookmark{'createtxg'};
+			if (($comparetxg && $bookmark{'createtxg'} > $snaps{'source'}{$matchingsnap}{'createtxg'}) ||
+				$bookmark{'creation'} > substr($snaps{'source'}{$matchingsnap}{'creation'}, 0, -3)) {
+				writelog('DEBUG', "using bookmark $bookmark{'name'} because it was created after latest matching snapshot $matchingsnap");
+				$matchingsnap = $bookmarksnap;
+				$usebookmark = 1;
 			}
 		}
 
@@ -676,7 +692,7 @@ sub syncdataset {
 
 			my $nextsnapshot = 0;
 
-			if (%bookmark) {
+			if ($usebookmark) {
 
 				if (!defined $args{'no-stream'}) {
 					# if intermediate snapshots are needed we need to find the next oldest snapshot,
@@ -736,7 +752,7 @@ sub syncdataset {
 
 			# do a normal replication if bookmarks aren't used or if previous
 			# bookmark replication was only done to the next oldest snapshot
-			if (!%bookmark || $nextsnapshot) {
+			if (!$usebookmark || $nextsnapshot) {
 				if ($matchingsnap eq $newsyncsnap) {
 					# edge case: bookmark replication used the latest snapshot
 					return 0;
@@ -801,7 +817,7 @@ sub syncdataset {
 
 	# if "--use-hold" parameter is used set hold on newsync snapshot and remove hold on matching snapshot both on source and target
 	# hold name: "syncoid" + identifier + hostname -> in case of replication to multiple targets separate holds can be set for each target by assinging different identifiers to each target. Only if all targets have been replicated all syncoid holds are removed from the matching snapshot and it can be removed
-	if (defined $args{'use-hold'}) {
+	if (defined $args{'use-hold'} && !$usebookmark) {
 		my $holdcmd;
 		my $holdreleasecmd;
 		my $hostid = hostname();
@@ -1964,6 +1980,7 @@ sub getbookmarks() {
 
 	for my $bookmark (keys %bookmark_data) {
 		my $guid = $bookmark_data{$bookmark}{'guid'};
+		$bookmarks{$guid}{'guid'} = $guid;
 		$bookmarks{$guid}{'name'} = $bookmark;
 		$bookmarks{$guid}{'creation'} = $bookmark_data{$bookmark}{'creation'};
 		$bookmarks{$guid}{'createtxg'} = $bookmark_data{$bookmark}{'createtxg'};