Share log mutations between backups and DRs which have the same backu… #3
Conversation
@@ -1141,7 +1273,7 @@ namespace dbBackup { | |||
return Void(); | |||
} | |||
|
|||
ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Key logUid, Key backupUid, Key keyAddPrefix, Key keyRemovePrefix, Key keyConfigBackupRanges, Key tagName, TaskCompletionKey completionKey, Reference<TaskFuture> waitFor = Reference<TaskFuture>(), bool databasesInSync=false) | |||
ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Key logUid, Key backupUid, /*Key destUid,*/ Key keyAddPrefix, Key keyRemovePrefix, Key keyConfigBackupRanges, Key tagName, TaskCompletionKey completionKey, Reference<TaskFuture> waitFor = Reference<TaskFuture>(), bool databasesInSync=false) |
etschannen
Feb 20, 2018
Contributor
remove comment
remove comment
@@ -1496,7 +1497,7 @@ namespace fileBackup { | |||
.detail("ScheduledVersion", scheduledVersion) | |||
.detail("BeginKey", range.begin.printable()) | |||
.detail("EndKey", range.end.printable()) | |||
.suppressFor(2); | |||
.suppressFor(2, true); |
etschannen
Feb 20, 2018
Contributor
unneeded
unneeded
Reference<Cluster> cluster = Cluster::createCluster(connFile, -1); | ||
Database cx = cluster->createDatabase(LiteralStringRef("DB")).get(); | ||
|
||
Reference<ClusterConnectionFile> extraFile(new ClusterConnectionFile(*g_simulator.extraDB)); | ||
Reference<Cluster> extraCluster = Cluster::createCluster(extraFile, -1); | ||
state Database extraDB = extraCluster->createDatabase(LiteralStringRef("DB")).get(); | ||
|
||
TraceEvent("StartingBackupAgents").detail("connFile", connFile->getConnectionString().toString()).detail("extraString", extraFile->getConnectionString().toString()); | ||
TraceEvent("StartingDrAgents").detail("connFile", connFile->getConnectionString().toString()).detail("extraString", extraFile->getConnectionString().toString()); |
etschannen
Feb 20, 2018
Contributor
add a similar trace event for backup agents
add a similar trace event for backup agents
Void _ = wait(delay(1.0)); | ||
} | ||
|
||
TraceEvent("StoppingBackupAgents"); | ||
TraceEvent("StoppingDrAgents"); | ||
|
etschannen
Feb 20, 2018
Contributor
add a similar trace event for backup
add a similar trace event for backup
shareLogRange=true | ||
performRestore=false | ||
|
||
testName=BackupToDBCorrectness |
etschannen
Feb 20, 2018
Contributor
something needs to restore
something needs to restore
} | ||
|
||
Future<Void> clearLogRanges(Reference<ReadYourWritesTransaction> tr, bool clearVersionHistory, Key logUidValue, Key destUidValue, Version beginVersion, Version endVersion) { | ||
return _clearLogRanges(tr, clearVersionHistory, logUidValue, destUidValue, beginVersion, endVersion); |
etschannen
Feb 21, 2018
Contributor
add a comment about the fact that endVersion - beginVersion should not be too large
add a comment about the fact that endVersion - beginVersion should not be too large
if (!v.present()) { | ||
return Void(); | ||
} | ||
|
etschannen
Feb 21, 2018
Contributor
use logUid instead of beginVersion for finding ourself in the list of versions
add an ASSERT_WE_THINK that v matches beginVersion, and matches ourself in the backupLatestVersions list
use logUid instead of beginVersion for finding ourself in the list of versions
add an ASSERT_WE_THINK that v matches beginVersion, and matches ourself in the backupLatestVersions list
etschannen
Feb 21, 2018
Contributor
do not get backupLatestVersionsKey separately; loop through backupVersions and check for ourself in those results to accomplish the same thing
do not get backupLatestVersionsKey separately; loop through backupVersions and check for ourself in those results to accomplish the same thing
if (currVersion > beginVersion) { | ||
if (currVersion < nextSmallestVersion) { | ||
nextSmallestVersion = currVersion; | ||
} |
etschannen
Feb 21, 2018
Contributor
use std::min for code clarity
use std::min for code clarity
@@ -609,7 +625,7 @@ namespace dbBackup { | |||
tr.addReadConflictRange(singleKeyRange(kv.key)); | |||
first = false; | |||
} | |||
tr.set(kv.key.removePrefix(backupLogKeys.begin).withPrefix(applyLogKeys.begin), kv.value); | |||
tr.set(kv.key.removePrefix(backupLogKeys.begin).removePrefix(task->params[BackupAgentBase::destUid]).withPrefix(task->params[BackupAgentBase::keyConfigLogUid]).withPrefix(applyLogKeys.begin), kv.value); | |||
bytesSet += kv.expectedSize() - backupLogKeys.begin.expectedSize() + applyLogKeys.begin.expectedSize(); |
etschannen
Feb 21, 2018
Contributor
calculate the two withPrefixs into a single string
use mutateString to directly change the bytes in the key instead of using withPrefix to copy the bytes into a new string
calculate the two withPrefixs into a single string
use mutateString to directly change the bytes in the key instead of using withPrefix to copy the bytes into a new string
beginVersion = BinaryReader::fromStringRef<Version>(bVersion.get(), Unversioned()); | ||
|
||
Void _ = wait(tr->commit()); | ||
endVersion = tr->getCommittedVersion(); | ||
break; |
etschannen
Feb 21, 2018
Contributor
for safety use committedVersion + 1
for safety use committedVersion + 1
Void _ = wait(tr->commit()); | ||
break; | ||
} catch (Error &e) { | ||
Void _ = wait(srcTr->onError(e)); |
etschannen
Feb 21, 2018
Contributor
tr not srcTr
tr not srcTr
|
||
Void _ = wait(srcTr->commit()); | ||
endVersion = srcTr->getCommittedVersion(); | ||
|
etschannen
Feb 21, 2018
Contributor
+1 committed version for safety
+1 committed version for safety
} | ||
} | ||
} | ||
|
etschannen
Feb 21, 2018
Contributor
factor this transaction into a separate function since it is used so many times
factor this transaction into a separate function since it is used so many times
79d6cb8
to
65a7a75
Should this PR be against the release-5.2 branch instead of the master branch? |
Share log mutations between backups and DRs which have the same backu…
/home/anoyes/workspace/foundationdb/fdbserver/Ratekeeper.actor.cpp:86:8: runtime error: load of value 1231493777, which is not a valid value for type 'limitReason_t' #0 0x310e961 in StorageQueueInfo::StorageQueueInfo(StorageQueueInfo&&) /home/anoyes/workspace/foundationdb/fdbserver/Ratekeeper.actor.cpp:86 #1 0x310eacd in MapPair<UID, StorageQueueInfo>::MapPair<UID, StorageQueueInfo>(UID&&, StorageQueueInfo&&) /home/anoyes/workspace/foundationdb/flow/IndexedSet.h:242 #2 0x310b35e in MapPair<std::decay<UID>::type, std::decay<StorageQueueInfo>::type> mapPair<UID, StorageQueueInfo>(UID&&, StorageQueueInfo&&) /home/anoyes/workspace/foundationdb/flow/IndexedSet.h:258 #3 0x30a8b79 in a_body1 /home/anoyes/workspace/foundationdb/fdbserver/Ratekeeper.actor.cpp:195 #4 0x309b529 in TrackStorageServerQueueInfoActor /home/anoyes/build/foundationdb/fdbserver/Ratekeeper.actor.g.cpp:495 #5 0x309b9be in trackStorageServerQueueInfo(RatekeeperData* const&, StorageServerInterface const&) /home/anoyes/workspace/foundationdb/fdbserver/Ratekeeper.actor.cpp:194 #6 0x30cff63 in a_body1loopBody1when1cont1 /home/anoyes/workspace/foundationdb/fdbserver/Ratekeeper.actor.cpp:303 #7 0x30cd9da in a_body1loopBody1when1when1 /home/anoyes/build/foundationdb/fdbserver/Ratekeeper.actor.g.cpp:1170 #8 0x30ed4dd in a_callback_fire /home/anoyes/build/foundationdb/fdbserver/Ratekeeper.actor.g.cpp:1185 #9 0x30e6d81 in fire /home/anoyes/workspace/foundationdb/flow/flow.h:998 #10 0x4df0dc in void SAV<Void>::send<Void>(Void&&) /home/anoyes/workspace/foundationdb/flow/flow.h:447 #11 0x959891 in void Promise<Void>::send<Void>(Void&&) const /home/anoyes/workspace/foundationdb/flow/flow.h:778 #12 0x7b4b018 in Sim2::execTask(Sim2::Task&) (/home/anoyes/build/foundationdb/bin/fdbserver+0x7b4b018) #13 0x7bf9168 in Sim2::RunLoopActorState<Sim2::RunLoopActor>::a_body1loopBody1cont1(Void const&, int) /home/anoyes/workspace/foundationdb/fdbrpc/sim2.actor.cpp:979 #14 0x7be7b68 in Sim2::RunLoopActorState<Sim2::RunLoopActor>::a_body1loopBody1when1(Void const&, int) /home/anoyes/build/foundationdb/fdbrpc/sim2.actor.g.cpp:5391 #15 0x7c329ff in Sim2::RunLoopActorState<Sim2::RunLoopActor>::a_callback_fire(ActorCallback<Sim2::RunLoopActor, 0, Void>*, Void) /home/anoyes/build/foundationdb/fdbrpc/sim2.actor.g.cpp:5406 #16 0x7c1fc73 in ActorCallback<Sim2::RunLoopActor, 0, Void>::fire(Void const&) /home/anoyes/workspace/foundationdb/flow/flow.h:998 #17 0x4df0dc in void SAV<Void>::send<Void>(Void&&) /home/anoyes/workspace/foundationdb/flow/flow.h:447 #18 0x959891 in void Promise<Void>::send<Void>(Void&&) const /home/anoyes/workspace/foundationdb/flow/flow.h:778 #19 0x7fe74a4 in N2::PromiseTask::operator()() /home/anoyes/workspace/foundationdb/flow/Net2.actor.cpp:481 #20 0x7fb6ff7 in N2::Net2::run() /home/anoyes/workspace/foundationdb/flow/Net2.actor.cpp:657 #21 0x7b71bd3 in Sim2::_runActorState<Sim2::_runActor>::a_body1(int) /home/anoyes/workspace/foundationdb/fdbrpc/sim2.actor.cpp:989 #22 0x7b2ee51 in Sim2::_runActor::_runActor(Sim2* const&) /home/anoyes/build/foundationdb/fdbrpc/sim2.actor.g.cpp:5608 #23 0x7b2f268 in Sim2::_run(Sim2* const&) /home/anoyes/workspace/foundationdb/fdbrpc/sim2.actor.cpp:987 #24 0x7b2f2c8 in Sim2::run() /home/anoyes/workspace/foundationdb/fdbrpc/sim2.actor.cpp:996 #25 0x21040a6 in main /home/anoyes/workspace/foundationdb/fdbserver/fdbserver.actor.cpp:1793 #26 0x7f03492ba504 in __libc_start_main (/lib64/libc.so.6+0x22504) #27 0x464914 (/home/anoyes/build/foundationdb/bin/fdbserver+0x464914)
/Users/anoyes/workspace/foundationdb/fdbserver/TLogInterface.h:149:8: runtime error: load of value 232, which is not a valid value for type 'bool' #0 0xc608fb in TLogPeekReply::TLogPeekReply(TLogPeekReply const&) /Users/anoyes/workspace/foundationdb/fdbserver/TLogInterface.h:149 #1 0x242bf87 in ILogSystem::ServerPeekCursor::ServerPeekCursor(TLogPeekReply const&, LogMessageVersion const&, LogMessageVersion const&, int, int, bool, long, Tag) /Users/anoyes/workspace/foundationdb/fdbserver/LogSystemPeekCursor.actor.cpp:35 #2 0x242da77 in ILogSystem::ServerPeekCursor::cloneNoMore() /Users/anoyes/workspace/foundationdb/fdbserver/LogSystemPeekCursor.actor.cpp:47 #3 0x24362d5 in ILogSystem::MergedPeekCursor::cloneNoMore() /Users/anoyes/workspace/foundationdb/fdbserver/LogSystemPeekCursor.actor.cpp:325 #4 0x244bf45 in ILogSystem::MultiCursor::cloneNoMore() /Users/anoyes/workspace/foundationdb/fdbserver/LogSystemPeekCursor.actor.cpp:838 #5 0x36b5a36 in a_body1cont5loopBody1 /Users/anoyes/workspace/foundationdb/fdbserver/storageserver.actor.cpp:2621 #6 0x36b3110 in a_body1cont5loopHead1 /Users/anoyes/build/foundationdb/fdbserver/storageserver.actor.g.cpp:8664 #7 0x36b07fe in a_body1cont5 /Users/anoyes/build/foundationdb/fdbserver/storageserver.actor.g.cpp:8576 #8 0x36abda8 in a_body1cont4when1 /Users/anoyes/build/foundationdb/fdbserver/storageserver.actor.g.cpp:8582 #9 0x36a8dc2 in a_body1cont4 /Users/anoyes/build/foundationdb/fdbserver/storageserver.actor.g.cpp:8454 #10 0x36a4bf6 in a_body1cont3break1 /Users/anoyes/build/foundationdb/fdbserver/storageserver.actor.g.cpp:8489 #11 0x36a2c01 in a_body1cont3loopBody1cont1 /Users/anoyes/build/foundationdb/fdbserver/storageserver.actor.g.cpp:8505 #12 0x369fd36 in a_body1cont3loopBody1when1 /Users/anoyes/build/foundationdb/fdbserver/storageserver.actor.g.cpp:8513 #13 0x3700dcb in a_callback_fire /Users/anoyes/build/foundationdb/fdbserver/storageserver.actor.g.cpp:8528 #14 0x36e5210 in fire /Users/anoyes/workspace/foundationdb/flow/flow.h:998 #15 0x4dfb2a in SAV<Void>::finishSendAndDelPromiseRef() /Users/anoyes/workspace/foundationdb/flow/flow.h:479 #16 0x2484b07 in a_body1loopBody1cont1 /Users/anoyes/build/foundationdb/fdbserver/LogSystemPeekCursor.actor.g.cpp:1526 #17 0x24822cf in a_body1loopBody1cont2 /Users/anoyes/build/foundationdb/fdbserver/LogSystemPeekCursor.actor.g.cpp:1535 #18 0x247e228 in a_body1loopBody1when1 /Users/anoyes/build/foundationdb/fdbserver/LogSystemPeekCursor.actor.g.cpp:1541 #19 0x249be87 in a_callback_fire /Users/anoyes/build/foundationdb/fdbserver/LogSystemPeekCursor.actor.g.cpp:1556 #20 0x249668f in fire /Users/anoyes/workspace/foundationdb/flow/flow.h:998 #21 0x4dfb2a in SAV<Void>::finishSendAndDelPromiseRef() /Users/anoyes/workspace/foundationdb/flow/flow.h:479 #22 0x80557e in a_body1when1 /Users/anoyes/build/foundationdb/flow/genericactors.actor.g.h:11591 #23 0x8916ef in a_callback_fire /Users/anoyes/build/foundationdb/flow/genericactors.actor.g.h:11620 #24 0x8735f5 in fire /Users/anoyes/workspace/foundationdb/flow/flow.h:998 #25 0x4dfb2a in SAV<Void>::finishSendAndDelPromiseRef() /Users/anoyes/workspace/foundationdb/flow/flow.h:479 #26 0x24820f8 in a_body1cont1loopBody1when1 /Users/anoyes/build/foundationdb/fdbserver/LogSystemPeekCursor.actor.g.cpp:860 #27 0x249c852 in a_callback_fire /Users/anoyes/build/foundationdb/fdbserver/LogSystemPeekCursor.actor.g.cpp:886 #28 0x249786c in fire /Users/anoyes/workspace/foundationdb/flow/flow.h:998 #29 0xc9d2dc in SAV<TLogPeekReply>::finishSendAndDelPromiseRef() /Users/anoyes/workspace/foundationdb/flow/flow.h:479 #30 0x248b39f in a_body1cont2 /Users/anoyes/build/foundationdb/flow/genericactors.actor.g.h:11858 #31 0x2489d02 in a_body1when1 /Users/anoyes/build/foundationdb/flow/genericactors.actor.g.h:11865 #32 0x249a150 in a_callback_fire /Users/anoyes/build/foundationdb/flow/genericactors.actor.g.h:11880 #33 0x2492a4f in fire /Users/anoyes/workspace/foundationdb/flow/flow.h:998 #34 0xc9d2dc in SAV<TLogPeekReply>::finishSendAndDelPromiseRef() /Users/anoyes/workspace/foundationdb/flow/flow.h:479 #35 0x248df9b in a_body1cont2 /Users/anoyes/build/foundationdb/fdbrpc/genericactors.actor.g.h:2762 #36 0x248b7da in a_body1when1 /Users/anoyes/build/foundationdb/fdbrpc/genericactors.actor.g.h:2769 #37 0x2499c88 in a_callback_fire /Users/anoyes/build/foundationdb/fdbrpc/genericactors.actor.g.h:2784 #38 0x2492371 in fire /Users/anoyes/workspace/foundationdb/flow/flow.h:998 #39 0xc9d2dc in SAV<TLogPeekReply>::finishSendAndDelPromiseRef() /Users/anoyes/workspace/foundationdb/flow/flow.h:479 #40 0xc60fb3 in void SAV<TLogPeekReply>::sendAndDelPromiseRef<TLogPeekReply&>(TLogPeekReply&) /Users/anoyes/workspace/foundationdb/flow/flow.h:472 #41 0xc1137a in NetSAV<TLogPeekReply>::receive(ArenaObjectReader&) /Users/anoyes/workspace/foundationdb/fdbrpc/fdbrpc.h:111 #42 0x78eda75 in a_body1cont1 /Users/anoyes/workspace/foundationdb/fdbrpc/FlowTransport.actor.cpp:652 #43 0x78f7967 in a_body1cont2 /Users/anoyes/build/foundationdb/fdbrpc/FlowTransport.actor.g.cpp:2369 #44 0x78ed4d8 in a_body1when1 /Users/anoyes/build/foundationdb/fdbrpc/FlowTransport.actor.g.cpp:2375 #45 0x791af45 in a_callback_fire /Users/anoyes/build/foundationdb/fdbrpc/FlowTransport.actor.g.cpp:2390 #46 0x7914670 in fire /Users/anoyes/workspace/foundationdb/flow/flow.h:998 #47 0x4df0dc in void SAV<Void>::send<Void>(Void&&) /Users/anoyes/workspace/foundationdb/flow/flow.h:447 #48 0x959891 in void Promise<Void>::send<Void>(Void&&) const /Users/anoyes/workspace/foundationdb/flow/flow.h:778 #49 0x7b4b022 in Sim2::execTask(Sim2::Task&) (/Users/anoyes/build/foundationdb/bin/fdbserver+0x7b4b022) #50 0x7bf9172 in Sim2::RunLoopActorState<Sim2::RunLoopActor>::a_body1loopBody1cont1(Void const&, int) /Users/anoyes/workspace/foundationdb/fdbrpc/sim2.actor.cpp:979 #51 0x7be7b72 in Sim2::RunLoopActorState<Sim2::RunLoopActor>::a_body1loopBody1when1(Void const&, int) /Users/anoyes/build/foundationdb/fdbrpc/sim2.actor.g.cpp:5391 #52 0x7c32a09 in Sim2::RunLoopActorState<Sim2::RunLoopActor>::a_callback_fire(ActorCallback<Sim2::RunLoopActor, 0, Void>*, Void) /Users/anoyes/build/foundationdb/fdbrpc/sim2.actor.g.cpp:5406 #53 0x7c1fc7d in ActorCallback<Sim2::RunLoopActor, 0, Void>::fire(Void const&) /Users/anoyes/workspace/foundationdb/flow/flow.h:998 #54 0x4df0dc in void SAV<Void>::send<Void>(Void&&) /Users/anoyes/workspace/foundationdb/flow/flow.h:447 #55 0x959891 in void Promise<Void>::send<Void>(Void&&) const /Users/anoyes/workspace/foundationdb/flow/flow.h:778 #56 0x7fe74ae in N2::PromiseTask::operator()() /Users/anoyes/workspace/foundationdb/flow/Net2.actor.cpp:481 #57 0x7fb7001 in N2::Net2::run() /Users/anoyes/workspace/foundationdb/flow/Net2.actor.cpp:657 #58 0x7b71bdd in Sim2::_runActorState<Sim2::_runActor>::a_body1(int) /Users/anoyes/workspace/foundationdb/fdbrpc/sim2.actor.cpp:989 #59 0x7b2ee5b in Sim2::_runActor::_runActor(Sim2* const&) /Users/anoyes/build/foundationdb/fdbrpc/sim2.actor.g.cpp:5608 #60 0x7b2f272 in Sim2::_run(Sim2* const&) /Users/anoyes/workspace/foundationdb/fdbrpc/sim2.actor.cpp:987 #61 0x7b2f2d2 in Sim2::run() /Users/anoyes/workspace/foundationdb/fdbrpc/sim2.actor.cpp:996 #62 0x2104064 in main /Users/anoyes/workspace/foundationdb/fdbserver/fdbserver.actor.cpp:1793 #63 0x7fb7c6561504 in __libc_start_main (/lib64/libc.so.6+0x22504) #64 0x464914 (/Users/anoyes/build/foundationdb/bin/fdbserver+0x464914)
…nowflake-tools SNOW-136449 Merge 6.2 into 6.2-snowflake-tools
Features/cache initialization
…e_tag transaction tagging in mako
…splitpoint More ParallelStream improvements
Some documentation for multi-threaded client
…p range