Merging staging/replication-2.0 into devel.

.github/workflows/clang-format.yml

@@ -2,7 +2,7 @@ name: clang-format
 on:
   workflow_dispatch:
   pull_request:
-    branches: [ devel ]
+    branches: [ devel , staging/replication-2.0 ]
     paths:
       - "arangod/**"
       - "client-tools/**"

Documentation/Metrics/arangodb_replication2_replicated_log_inserts_rtt.yaml

@@ -12,4 +12,4 @@ exposedBy:
 description: |
   Inserts into replicated logs this server is a leader of are counted by this
   histogram, plus their respective round-trip time. This includes the time for
-  replication, at least until the writeConcern is satisified.
+  replication, at least until the writeConcern is satisfied.

Documentation/Metrics/arangodb_replication2_replicated_log_number_accepted_entries_total.yaml

@@ -0,0 +1,12 @@
+name: arangodb_replication2_replicated_log_number_accepted_entries_total
+introducedIn: "3.10.0"
+help: |
+  Number of accepted (not yet committed) log entries.
+unit: number
+type: counter
+category: Replication
+complexity: simple
+exposedBy:
+  - dbserver
+description: |
+  The entries have been inserted into the log and are being replicated but a quorum has not yet been reached.

Documentation/Metrics/arangodb_replication2_replicated_log_number_committed_entries_total.yaml

@@ -0,0 +1,12 @@
+name: arangodb_replication2_replicated_log_number_committed_entries_total
+introducedIn: "3.10.0"
+help: |
+  Number of committed log entries.
+unit: number
+type: counter
+category: Replication
+complexity: simple
+exposedBy:
+  - dbserver
+description: |
+  Number of committed log entries.

Documentation/Metrics/arangodb_replication2_replicated_log_number_compacted_entries_total.yaml

@@ -0,0 +1,13 @@
+name: arangodb_replication2_replicated_log_number_compacted_entries_total
+introducedIn: "3.10.0"
+help: |
+  Number of compacted log entries.
+unit: number
+type: counter
+category: Replication
+complexity: simple
+exposedBy:
+  - dbserver
+description: |
+  Number of compacted log entries. If log entries have been replicated to every participant and applied
+  durably, they can be compacted, i.e. deleted.

Documentation/Metrics/arangodb_replication2_replicated_log_number_meta_entries_total.yaml

@@ -0,0 +1,13 @@
+name: arangodb_replication2_replicated_log_number_meta_entries_total
+introducedIn: "3.10.0"
+help: |
+  Number of meta log entries.
+unit: number
+type: counter
+category: Replication
+complexity: simple
+exposedBy:
+  - dbserver
+description: |
+  Number of meta log entries. A meta log entry is used to create artificial write barriers, for example during a
+  configuration change or move shard operation.

arangod/Agency/CleanOutServer.cpp

@@ -24,6 +24,7 @@
 #include "CleanOutServer.h"
 
 #include "Agency/AgentInterface.h"
+#include "Agency/Helpers.h"
 #include "Agency/Job.h"
 #include "Agency/JobContext.h"
 #include "Agency/MoveShard.h"
@@ -370,11 +371,17 @@ bool CleanOutServer::start(bool& aborts) {
 bool CleanOutServer::scheduleMoveShards(std::shared_ptr<Builder>& trx) {
   std::vector<std::string> servers = availableServers(_snapshot);
 
+  Node::Children const& databaseProperties =
+      _snapshot.hasAsChildren(planDBPrefix).value().get();
   Node::Children const& databases =
-      _snapshot.hasAsChildren("/Plan/Collections").value().get();
+      _snapshot.hasAsChildren(planColPrefix).value().get();
   size_t sub = 0;
 
   for (auto const& database : databases) {
+    if (isReplicationTwoDB(databaseProperties, database.first)) {
+      continue;
+    }
+
     // Find shardsLike dependencies
     for (auto const& collptr : database.second->children()) {
       auto const& collection = *(collptr.second);

arangod/Agency/FailedServer.cpp

@@ -27,6 +27,7 @@
 #include "Agency/AgentInterface.h"
 #include "Agency/FailedFollower.h"
 #include "Agency/FailedLeader.h"
+#include "Agency/Helpers.h"
 #include "Agency/Job.h"
 #include "Basics/StaticStrings.h"
 
@@ -151,16 +152,18 @@ bool FailedServer::start(bool& aborts) {
       VPackObjectBuilder oper(transactions.get());
       // Add pending
 
+      auto const& databaseProperties =
+          _snapshot.hasAsChildren(planDBPrefix).value().get();
       auto const& databases =
-          _snapshot.hasAsChildren("/Plan/Collections").value().get();
-      // auto const& current =
-      // _snapshot.hasAsChildren("/Current/Collections").first;
+          _snapshot.hasAsChildren(planColPrefix).value().get();
 
       size_t sub = 0;
 
       // FIXME: looks OK, but only the non-clone shards are put into the job
       for (auto const& database : databases) {
-        // dead code   auto cdatabase = current.at(database.first)->children();
+        if (isReplicationTwoDB(databaseProperties, database.first)) {
+          continue;
+        }
 
         for (auto const& collptr : database.second->children()) {
           auto const& collection = *(collptr.second);

arangod/Agency/Helpers.cpp

@@ -0,0 +1,47 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2022 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Manuel Pöter
+////////////////////////////////////////////////////////////////////////////////
+
+#include "Agency/Helpers.h"
+
+#include "Basics/StaticStrings.h"
+#include "Replication2/Version.h"
+
+namespace arangodb::consensus {
+
+bool isReplicationTwoDB(Node::Children const& databases,
+                        std::string const& dbName) {
+  auto it = databases.find(dbName);
+  if (it == databases.end()) {
+    // this should actually never happen, but if it does we simply claim that
+    // this is an old replication 1 DB.
+    return false;
+  }
+
+  if (auto v = it->second->hasAsString(StaticStrings::ReplicationVersion); v) {
+    auto res = replication::parseVersion(v.value());
+    return res.ok() && res.get() == replication::Version::TWO;
+  }
+  return false;
+}
+
+}  // namespace arangodb::consensus

arangod/Agency/Helpers.h

@@ -0,0 +1,33 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2022 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Manuel Pöter
+////////////////////////////////////////////////////////////////////////////////
+
+#pragma once
+
+#include "Agency/Node.h"
+
+namespace arangodb::consensus {
+
+bool isReplicationTwoDB(Node::Children const& databases,
+                        std::string const& dbName);
+
+}  // namespace arangodb::consensus

arangod/Agency/ResignLeadership.cpp

@@ -25,6 +25,7 @@
 #include "Agency/ResignLeadership.h"
 
 #include "Agency/AgentInterface.h"
+#include "Agency/Helpers.h"
 #include "Agency/Job.h"
 #include "Agency/JobContext.h"
 #include "Agency/MoveShard.h"
@@ -361,11 +362,17 @@ bool ResignLeadership::start(bool& aborts) {
 bool ResignLeadership::scheduleMoveShards(std::shared_ptr<Builder>& trx) {
   std::vector<std::string> servers = availableServers(_snapshot);
 
+  Node::Children const& databaseProperties =
+      _snapshot.hasAsChildren(planDBPrefix).value().get();
   Node::Children const& databases =
-      _snapshot.hasAsChildren("/Plan/Collections").value().get();
+      _snapshot.hasAsChildren(planColPrefix).value().get();
   size_t sub = 0;
 
   for (auto const& database : databases) {
+    if (isReplicationTwoDB(databaseProperties, database.first)) {
+      continue;
+    }
+
     // Find shardsLike dependencies
     for (auto const& collptr : database.second->children()) {
       auto const& collection = *(collptr.second);

arangod/Agency/Supervision.cpp

@@ -32,6 +32,7 @@
 #include "Agency/Agent.h"
 #include "Agency/CleanOutServer.h"
 #include "Agency/FailedServer.h"
+#include "Agency/Helpers.h"
 #include "Agency/Job.h"
 #include "Agency/JobContext.h"
 #include "Agency/RemoveFollower.h"
@@ -2982,8 +2983,14 @@ void arangodb::consensus::enforceReplicationFunctional(
 
   // We will loop over plannedDBs, so we use hasAsChildren
   auto const& plannedDBs = snapshot.hasAsChildren(planColPrefix).value().get();
+  auto const& databaseProperties =
+      snapshot.hasAsChildren("/Plan/Databases").value().get();
 
   for (const auto& db_ : plannedDBs) {  // Planned databases
+    if (isReplicationTwoDB(databaseProperties, db_.first)) {
+      continue;
+    }
+
     auto const& db = *(db_.second);
     for (const auto& col_ : db.children()) {  // Planned collections
       auto const& col = *(col_.second);

arangod/CMakeLists.txt

@@ -625,8 +625,13 @@ set(LIB_ARANGO_REPLICATION2_SOURCES
   Replication2/ReplicatedState/UpdateReplicatedState.h
   Replication2/StateMachines/BlackHole/BlackHoleStateMachine.cpp
   Replication2/StateMachines/BlackHole/BlackHoleStateMachineFeature.cpp
+  Replication2/StateMachines/Document/DocumentCore.cpp
+  Replication2/StateMachines/Document/DocumentFollowerState.cpp
+  Replication2/StateMachines/Document/DocumentLeaderState.cpp
+  Replication2/StateMachines/Document/DocumentLogEntry.cpp
   Replication2/StateMachines/Document/DocumentStateMachine.cpp
   Replication2/StateMachines/Document/DocumentStateMachineFeature.cpp
+  Replication2/StateMachines/Document/DocumentStateStrategy.cpp
   Replication2/StateMachines/Prototype/PrototypeLogEntry.cpp
   Replication2/StateMachines/Prototype/PrototypeLeaderState.cpp
   Replication2/StateMachines/Prototype/PrototypeCore.cpp
@@ -665,6 +670,7 @@ set(LIB_ARANGO_AGENCY_SOURCES
   Agency/FailedFollower.cpp
   Agency/FailedLeader.cpp
   Agency/FailedServer.cpp
+  Agency/Helpers.cpp
   Agency/Inception.cpp
   Agency/Job.cpp
   Agency/JobContext.cpp

arangod/Cluster/AgencyCache.cpp

@@ -77,6 +77,17 @@ bool AgencyCache::start() {
   return true;
 }
 
+consensus::index_t AgencyCache::get(
+    arangodb::velocypack::Builder& result,
+    std::shared_ptr<const cluster::paths::Path> const& path) const {
+  result.clear();
+  std::shared_lock g(_storeLock);
+  if (_commitIndex > 0) {
+    _readDB.get(path->str(), result, false);
+  }
+  return _commitIndex;
+}
+
 // Fill existing Builder from readDB, mainly /Plan /Current
 index_t AgencyCache::get(VPackBuilder& result, std::string const& path) const {
   result.clear();

arangod/Cluster/AgencyCache.h

@@ -82,6 +82,11 @@ class AgencyCache final : public ServerThread<ArangodServer> {
   /// prepended
   consensus::query_t dump() const;
 
+  /// @brief Get velocypack from node downward.
+  consensus::index_t get(
+      arangodb::velocypack::Builder& result,
+      std::shared_ptr<const cluster::paths::Path> const& path) const;
+
   /// @brief Get velocypack from node downward. AgencyCommHelper::path is
   /// prepended
   consensus::index_t get(arangodb::velocypack::Builder& result,