Merge pull request #539 from AlCutter/wait_for_data_when_joining_exis…

…ting_cluster Wait for DB to catch up to ServingSTH at boot.
google · Feb 27, 2015 · 4eed1a7 · 4eed1a7
2 parents 6d5680c + 6b78b42
commit 4eed1a7
Showing 1 changed file with 31 additions and 15 deletions.
diff --git a/cpp/server/ct-server.cc b/cpp/server/ct-server.cc
@@ -292,6 +292,37 @@ int main(int argc, char* argv[]) {
                                                  etcd_client.get(), &election,
                                                  "/root", node_id));
 
+  const unique_ptr<ContinuousFetcher> fetcher(
+      ContinuousFetcher::New(event_base.get(), db));
+
+  ClusterStateController<LoggedCertificate> cluster_controller(
+      &internal_pool, event_base, &url_fetcher, db, &consistent_store,
+      &election, fetcher.get());
+
+  // Publish this node's hostname:port info
+  cluster_controller.SetNodeHostPort(FLAGS_server, FLAGS_port);
+  {
+    ct::SignedTreeHead db_sth;
+    if (db->LatestTreeHead(&db_sth) ==
+        Database<LoggedCertificate>::LOOKUP_OK) {
+      cluster_controller.NewTreeHead(db_sth);
+    }
+  }
+
+  // If we're joining an existing cluster, this node needs to get its database
+  // up-to-date with the serving_sth before we can do anything, so we'll wait
+  // here for that:
+  util::StatusOr<ct::SignedTreeHead> serving_sth(
+      consistent_store.GetServingSTH());
+  if (serving_sth.ok()) {
+    while (db->TreeSize() < serving_sth.ValueOrDie().tree_size()) {
+      LOG(WARNING) << "Waiting for local database to catch up to serving_sth ("
+                   << db->TreeSize() << " of "
+                   << serving_sth.ValueOrDie().tree_size() << ")";
+      sleep(1);
+    }
+  }
+
   TreeSigner<LoggedCertificate> tree_signer(std::chrono::duration<double>(
                                                 FLAGS_guard_window_seconds),
                                             db, &consistent_store,
@@ -333,21 +364,6 @@ int main(int argc, char* argv[]) {
     CHECK(!FLAGS_server.empty());
   }
 
-  const unique_ptr<ContinuousFetcher> fetcher(
-      ContinuousFetcher::New(event_base.get(), db));
-  ClusterStateController<LoggedCertificate> cluster_controller(
-      &internal_pool, event_base, &url_fetcher, db, &consistent_store,
-      &election, fetcher.get());
-  // Publish this node's hostname:port info
-  cluster_controller.SetNodeHostPort(FLAGS_server, FLAGS_port);
-  {
-    ct::SignedTreeHead db_sth;
-    if (db->LatestTreeHead(&db_sth) ==
-        Database<LoggedCertificate>::LOOKUP_OK) {
-      cluster_controller.NewTreeHead(db_sth);
-    }
-  }
-
   // Just separate this out from the lambda below to try to be clear that this
   // is the *only* interaction with TreeSigner that we're allowing there.
   const function<SignedTreeHead(void)> get_latest_sth(