Merge branch 'devel' of https://github.com/arangodb/arangodb into fea…

…ture/planning-query-result-cache

* 'devel' of https://github.com/arangodb/arangodb:
  fix compile warning in non-maintainer mode
  windows implementation was missing
  Feature/cluster inventory version (#3152)
  fixed some MSVC warnings
  rename scripts
  change to devel

Installation/Pipeline/Jenkinsfile.groovy

@@ -142,7 +142,7 @@ def checkoutCommunity() {
                 poll: false,
                 scm: [
                     $class: 'GitSCM',
-                    branches: [[name: "*/feature/improve-jenkins"]],
+                    branches: [[name: "*/${sourceBranchLabel}"]],
                     doGenerateSubmoduleConfigurations: false,
                     extensions: [],
                     submoduleCfg: [],

arangod/Aql/ExecutionEngine.cpp

@@ -909,7 +909,9 @@ struct CoordinatorInstanciator : public WalkerWorker<ExecutionNode> {
     Serv2ColMap mappingServerToCollections;
     size_t length = edges.size();
 
+#ifdef USE_ENTERPRISE
     transaction::Methods* trx = query->trx();
+#endif
 
     auto findServerLists = [&] (ShardID const& shard) -> Serv2ColMap::iterator {
       auto serverList = clusterInfo->getResponsibleServer(shard);

arangod/Aql/IndexBlock.cpp

@@ -397,11 +397,11 @@ bool IndexBlock::skipIndex(size_t atMost) {
       THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
     }
 
-    uint64_t returned = (uint64_t) _returned;
-    bool ok = _cursor->skip(atMost - returned, returned);
-    _returned = (size_t) returned;
+    uint64_t returned = static_cast<uint64_t>(_returned);
+    int res = _cursor->skip(atMost - returned, returned);
+    _returned = static_cast<size_t>(returned);
 
-    if (ok) {
+    if (res == TRI_ERROR_NO_ERROR) {
       // We have skipped enough.
       // And this index could return more.
       // We are good.

arangod/Cache/Cache.cpp

@@ -270,7 +270,7 @@ bool Cache::reclaimMemory(uint64_t size) {
   return underLimit;
 }
 
-uint32_t Cache::hashKey(void const* key, uint32_t keySize) const {
+uint32_t Cache::hashKey(void const* key, size_t keySize) const {
   return (std::max)(static_cast<uint32_t>(1),
                     fasthash32(key, keySize, 0xdeadbeefUL));
 }

arangod/Cache/Cache.h

@@ -189,7 +189,7 @@ class Cache : public std::enable_shared_from_this<Cache> {
   static void freeValue(CachedValue* value);
   bool reclaimMemory(uint64_t size);
 
-  uint32_t hashKey(void const* key, uint32_t keySize) const;
+  uint32_t hashKey(void const* key, size_t keySize) const;
   void recordStat(Stat stat);
 
   bool reportInsert(bool hadEviction);

arangod/Cluster/ClusterInfo.cpp

@@ -108,79 +108,15 @@ static std::string extractErrorMessage(std::string const& shardId,
 /// @brief creates an empty collection info object
 ////////////////////////////////////////////////////////////////////////////////
 
-CollectionInfoCurrent::CollectionInfoCurrent() {}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief creates a collection info object from json
-////////////////////////////////////////////////////////////////////////////////
-
-CollectionInfoCurrent::CollectionInfoCurrent(ShardID const& shardID,
-                                             VPackSlice slice) {
-  add(shardID, slice);
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief creates a collection info object from another
-////////////////////////////////////////////////////////////////////////////////
-
-CollectionInfoCurrent::CollectionInfoCurrent(CollectionInfoCurrent const& other)
-    : _vpacks(other._vpacks) {
-  copyAllVPacks();
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief moves a collection info current object from another
-////////////////////////////////////////////////////////////////////////////////
-
-CollectionInfoCurrent::CollectionInfoCurrent(CollectionInfoCurrent&& other) {
-  _vpacks.swap(other._vpacks);
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief copy assigns a collection info current object from another one
-////////////////////////////////////////////////////////////////////////////////
-
-CollectionInfoCurrent& CollectionInfoCurrent::operator=(
-    CollectionInfoCurrent const& other) {
-  if (this == &other) {
-    return *this;
-  }
-  _vpacks = other._vpacks;
-  copyAllVPacks();
-  return *this;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief creates a collection info object from json
-////////////////////////////////////////////////////////////////////////////////
-
-CollectionInfoCurrent& CollectionInfoCurrent::operator=(
-    CollectionInfoCurrent&& other) {
-  if (this == &other) {
-    return *this;
-  }
-  _vpacks.clear();
-  _vpacks.swap(other._vpacks);
-  return *this;
-}
+CollectionInfoCurrent::CollectionInfoCurrent(uint64_t currentVersion)
+  : _currentVersion(currentVersion) {}
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief destroys a collection info object
 ////////////////////////////////////////////////////////////////////////////////
 
 CollectionInfoCurrent::~CollectionInfoCurrent() {}
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief copy slices behind the pointers in the map _vpacks
-////////////////////////////////////////////////////////////////////////////////
-void CollectionInfoCurrent::copyAllVPacks() {
-  for (auto it : _vpacks) {
-    auto builder = std::make_shared<VPackBuilder>();
-    builder->add(it.second->slice());
-    it.second = builder;
-  }
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief create the clusterinfo instance
 ////////////////////////////////////////////////////////////////////////////////
@@ -201,7 +137,8 @@ ClusterInfo* ClusterInfo::instance() { return _instance.get(); }
 ////////////////////////////////////////////////////////////////////////////////
 
 ClusterInfo::ClusterInfo(AgencyCallbackRegistry* agencyCallbackRegistry)
-    : _agency(), _agencyCallbackRegistry(agencyCallbackRegistry), _uniqid() {
+    : _agency(), _agencyCallbackRegistry(agencyCallbackRegistry),
+      _planVersion(0), _currentVersion(0), _uniqid() {
   _uniqid._currentValue = 1ULL;
   _uniqid._upperValue = 0ULL;
 
@@ -450,6 +387,19 @@ void ClusterInfo::loadPlan() {
     VPackSlice planSlice = planBuilder->slice();
 
     if (planSlice.isObject()) {
+      uint64_t newPlanVersion = 0;
+      VPackSlice planVersionSlice = planSlice.get("Version");
+      if (planVersionSlice.isNumber()) {
+        try {
+          newPlanVersion = planVersionSlice.getNumber<uint64_t>();
+        } catch (...) {
+        }
+      }
+      if (newPlanVersion == 0) {
+        LOG_TOPIC(WARN, Logger::CLUSTER)
+          << "Attention: /arango/Plan/Version in the agency is not set or not "
+             "a positive number.";
+      }
       decltype(_plannedDatabases) newDatabases;
       decltype(_plannedCollections) newCollections; // map<string /*database id*/
                                                     //    ,map<string /*collection id*/
@@ -590,6 +540,7 @@ void ClusterInfo::loadPlan() {
                     vocbase, collectionSlice);
               }
 #endif
+              newCollection->setPlanVersion(newPlanVersion);
               std::string const collectionName = newCollection->name();
               if (isCoordinator && !selectivityEstimates.empty()){
                 LOG_TOPIC(TRACE, Logger::CLUSTER) << "copy index estimates";
@@ -662,6 +613,7 @@ void ClusterInfo::loadPlan() {
 
       WRITE_LOCKER(writeLocker, _planProt.lock);
       _plan = planBuilder;
+      _planVersion = newPlanVersion;
       if (swapDatabases) {
         _plannedDatabases.swap(newDatabases);
       }
@@ -718,6 +670,20 @@ void ClusterInfo::loadCurrent() {
     VPackSlice currentSlice = currentBuilder->slice();
 
     if (currentSlice.isObject()) {
+      uint64_t newCurrentVersion = 0;
+      VPackSlice currentVersionSlice = currentSlice.get("Version");
+      if (currentVersionSlice.isNumber()) {
+        try {
+          newCurrentVersion = currentVersionSlice.getNumber<uint64_t>();
+        } catch (...) {
+        }
+      }
+      if (newCurrentVersion == 0) {
+        LOG_TOPIC(WARN, Logger::CLUSTER)
+          << "Attention: /arango/Current/Version in the agency is not set or "
+             "not a positive number.";
+      }
+
       decltype(_currentDatabases) newDatabases;
       decltype(_currentCollections) newCollections;
       decltype(_shardIds) newShardIds;
@@ -758,7 +724,7 @@ void ClusterInfo::loadCurrent() {
             std::string const collectionName = collectionSlice.key.copyString();
 
             auto collectionDataCurrent =
-                std::make_shared<CollectionInfoCurrent>();
+                std::make_shared<CollectionInfoCurrent>(newCurrentVersion);
             for (auto const& shardSlice :
                  VPackObjectIterator(collectionSlice.value)) {
               std::string const shardID = shardSlice.key.copyString();
@@ -788,6 +754,7 @@ void ClusterInfo::loadCurrent() {
       // Now set the new value:
       WRITE_LOCKER(writeLocker, _currentProt.lock);
       _current = currentBuilder;
+      _currentVersion = newCurrentVersion;
       if (swapDatabases) {
         _currentDatabases.swap(newDatabases);
       }
@@ -929,7 +896,7 @@ std::shared_ptr<CollectionInfoCurrent> ClusterInfo::getCollectionCurrent(
     loadCurrent();
   }
 
-  return std::make_shared<CollectionInfoCurrent>();
+  return std::make_shared<CollectionInfoCurrent>(0);
 }
 
 ////////////////////////////////////////////////////////////////////////////////

arangod/Cluster/ClusterInfo.h

@@ -56,23 +56,18 @@ class CollectionInfoCurrent {
   friend class ClusterInfo;
 
  public:
-  CollectionInfoCurrent();
+  CollectionInfoCurrent(uint64_t currentVersion);
 
-  CollectionInfoCurrent(ShardID const&, VPackSlice);
+  CollectionInfoCurrent(CollectionInfoCurrent const&) = delete;
 
-  CollectionInfoCurrent(CollectionInfoCurrent const&);
+  CollectionInfoCurrent(CollectionInfoCurrent&&) = delete;
 
-  CollectionInfoCurrent(CollectionInfoCurrent&&);
+  CollectionInfoCurrent& operator=(CollectionInfoCurrent const&) = delete;
 
-  CollectionInfoCurrent& operator=(CollectionInfoCurrent const&);
-
-  CollectionInfoCurrent& operator=(CollectionInfoCurrent&&);
+  CollectionInfoCurrent& operator=(CollectionInfoCurrent&&) = delete;
 
   ~CollectionInfoCurrent();
 
- private:
-  void copyAllVPacks();
-
  public:
   bool add(ShardID const& shardID, VPackSlice slice) {
     auto it = _vpacks.find(shardID);
@@ -177,6 +172,14 @@ class CollectionInfoCurrent {
     return std::string();
   }
 
+  //////////////////////////////////////////////////////////////////////////////
+  /// @brief get version that underlies this info in Current in the agency
+  //////////////////////////////////////////////////////////////////////////////
+
+  uint64_t getCurrentVersion() const {
+    return _currentVersion;
+  }
+
   //////////////////////////////////////////////////////////////////////////////
   /// @brief local helper to return boolean flags
   //////////////////////////////////////////////////////////////////////////////
@@ -206,6 +209,9 @@ class CollectionInfoCurrent {
 
  private:
   std::unordered_map<ShardID, std::shared_ptr<VPackBuilder>> _vpacks;
+
+  uint64_t _currentVersion;    // Version of Current in the agency that
+                               // underpins the data presented in this object
 };
 
 class ClusterInfo {
@@ -609,6 +615,12 @@ class ClusterInfo {
 
   ProtectionData _planProt;
 
+  uint64_t _planVersion;   // This is the version in the Plan which underlies
+                           // the data in _plannedCollections, _shards and
+                           // _shardKeys
+  uint64_t _currentVersion;  // This is the version in Current which underlies
+                             // the data in _currentDatabases,
+                             // _currentCollections and _shardsIds
   std::unordered_map<DatabaseID,
                      std::unordered_map<ServerID, VPackSlice>>
       _currentDatabases;  // from Current/Databases

arangod/Cluster/v8-cluster.cpp

@@ -706,6 +706,8 @@ static void JS_GetCollectionInfoCurrentClusterInfo(
       ClusterInfo::instance()->getCollectionCurrent(TRI_ObjectToString(args[0]),
                                                     cid);
 
+  result->Set(TRI_V8_ASCII_STRING("currentVersion"),
+              v8::Number::New(isolate, (double) cic->getCurrentVersion()));
   result->Set(TRI_V8_ASCII_STRING("type"),
               v8::Number::New(isolate, (int)ci->type()));
 

arangod/RestHandler/RestReplicationHandler.cpp

@@ -28,6 +28,7 @@
 #include "Aql/Query.h"
 #include "Aql/QueryRegistry.h"
 #include "Cluster/ClusterComm.h"
+#include "Cluster/ClusterHelpers.h"
 #include "Cluster/ClusterMethods.h"
 #include "Replication/InitialSyncer.h"
 #include "RestServer/QueryRegistryFeature.h"
@@ -476,7 +477,22 @@ void RestReplicationHandler::handleCommandClusterInventory() {
   resultBuilder.add(VPackValue("collections"));
   resultBuilder.openArray();
   for (auto const& c : cols) {
-    c->toVelocyPackForClusterInventory(resultBuilder, includeSystem);
+    // We want to check if the collection is usable and all followers
+    // are in sync:
+    auto shardMap = c->shardIds();
+      // shardMap is an unordered_map from ShardId (string) to a vector of
+      // servers (strings), wrapped in a shared_ptr
+    auto cic = ci->getCollectionCurrent(dbName,
+                                        basics::StringUtils::itoa(c->cid()));
+    // Check all shards:
+    bool isReady = true;
+    for (auto const& p : *shardMap) {
+      auto currentServerList = cic->servers(p.first  /* shardId */);
+      if (!ClusterHelpers::compareServerLists(p.second, currentServerList)) {
+        isReady = false;
+      }
+    }
+    c->toVelocyPackForClusterInventory(resultBuilder, includeSystem, isReady);
   }
   resultBuilder.close();  // collections
   TRI_voc_tick_t tick = TRI_CurrentTickServer();

arangod/RocksDBEngine/RocksDBEngine.cpp

@@ -1364,7 +1364,9 @@ Result RocksDBEngine::dropDatabase(TRI_voc_tick_t id) {
     }
   }
 
+#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
   size_t numDocsLeft = 0;
+#endif
 
   // remove collections
   for (auto const& val : collectionKVPairs(id)) {

arangod/VocBase/LogicalCollection.cpp

@@ -166,7 +166,8 @@ LogicalCollection::LogicalCollection(LogicalCollection const& other)
       _keyOptions(other._keyOptions),
       _keyGenerator(KeyGenerator::factory(VPackSlice(keyOptions()))),
       _physical(other.getPhysical()->clone(this)),
-      _clusterEstimateTTL(0) {
+      _clusterEstimateTTL(0),
+      _planVersion(other._planVersion) {
   TRI_ASSERT(_physical != nullptr);
   if (ServerState::instance()->isDBServer() ||
       !ServerState::instance()->isRunningInCluster()) {
@@ -206,7 +207,8 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase,
       _keyGenerator(),
       _physical(
           EngineSelectorFeature::ENGINE->createPhysicalCollection(this, info)),
-      _clusterEstimateTTL(0) {
+      _clusterEstimateTTL(0),
+      _planVersion(0) {
   // add keyoptions from slice
   TRI_ASSERT(info.isObject());
   VPackSlice keyOpts = info.get("keyOptions");
@@ -795,7 +797,8 @@ void LogicalCollection::setStatus(TRI_vocbase_col_status_e status) {
 }
 
 void LogicalCollection::toVelocyPackForClusterInventory(VPackBuilder& result,
-                                                        bool useSystem) const {
+                                                        bool useSystem,
+                                                        bool isReady) const {
   if (_isSystem && !useSystem) {
     return;
   }
@@ -823,6 +826,8 @@ void LogicalCollection::toVelocyPackForClusterInventory(VPackBuilder& result,
 
   result.add(VPackValue("indexes"));
   getIndexesVPack(result, false, false);
+  result.add("planVersion", VPackValue(getPlanVersion()));
+  result.add("isReady", VPackValue(isReady));
   result.close();  // CollectionInfo
 }