Feature/enterprise graphs (#16309)

CHANGELOG

@@ -1,6 +1,13 @@
 devel
 -----
 
+* Added Enterprise Graph feature to enterprise version of ArangoDB.
+  The enterprise graph is another graph sharding model that we introduced,
+  it is less strict, and therefore easier to start with, then SmartGraphs,
+  as it does not require a smartGraphAttribute, and allows free choice of
+  vertex _key values. But still maintains performance gains as compared to
+  general-graphs. For more details please check documentation.
+
 * APM-135: Added multithreading to assigning non-unique indexes to documents,
   in foreground or background mode. The number of index creation threads 
   is hardcoded to 2 for now. Improvements for higher parallelism are expected

arangod/Aql/Functions.cpp

@@ -9252,7 +9252,7 @@ AqlValue Functions::SelectSmartDistributeGraphInput(
   auto fromId = input.get(StaticStrings::IdString).stringView();
   auto res =
       SmartGraphValidationHelper::SmartValidationResult::validateVertexId(
-          fromId);
+          fromId, expressionContext->vocbase());
   if (res.ok()) {
     return AqlValue{input};
   }

arangod/Graph/Graph.cpp

@@ -779,6 +779,8 @@ void Graph::verticesToVpack(VPackBuilder& builder) const {
 
 bool Graph::isSmart() const { return false; }
 
+bool Graph::hasSmartGraphAttribute() const { return false; }
+
 bool Graph::isDisjoint() const { return false; }
 
 bool Graph::isSatellite() const { return _isSatellite; }

arangod/Graph/Graph.h

@@ -210,6 +210,7 @@ class Graph {
       std::string const& collectionName) const;
 
   virtual bool isSmart() const;
+  virtual bool hasSmartGraphAttribute() const;
   virtual bool isDisjoint() const;
   virtual bool isSatellite() const;
 

arangod/Graph/Providers/ClusterProvider.cpp

@@ -557,6 +557,14 @@ arangodb::transaction::Methods* ClusterProvider<StepImpl>::trx() {
   return _trx.get();
 }
 
+template<class Step>
+TRI_vocbase_t const& ClusterProvider<Step>::vocbase() const {
+  TRI_ASSERT(_trx != nullptr);
+  TRI_ASSERT(_trx->state() != nullptr);
+  TRI_ASSERT(_trx->transactionContextPtr() != nullptr);
+  return _trx.get()->vocbase();
+}
+
 template<class StepImpl>
 arangodb::aql::TraversalStats ClusterProvider<StepImpl>::stealStats() {
   auto t = _stats;

arangod/Graph/Providers/ClusterProvider.h

@@ -114,6 +114,7 @@ class ClusterProvider {
   void destroyEngines();
 
   [[nodiscard]] transaction::Methods* trx();
+  [[nodiscard]] TRI_vocbase_t const& vocbase() const;
 
   void prepareIndexExpressions(aql::Ast* ast);
 

arangod/Graph/Providers/ProviderTracer.cpp

@@ -209,6 +209,14 @@ transaction::Methods* ProviderTracer<ProviderImpl>::trx() {
   return _impl.trx();
 }
 
+template<class ProviderImpl>
+TRI_vocbase_t const& ProviderTracer<ProviderImpl>::vocbase() const {
+  double start = TRI_microtime();
+  auto sg = arangodb::scopeGuard(
+      [&]() noexcept { _stats["vocbase"].addTiming(TRI_microtime() - start); });
+  return _impl.vocbase();
+}
+
 template<class ProviderImpl>
 void ProviderTracer<ProviderImpl>::prepareContext(aql::InputAqlItemRow input) {
   double start = TRI_microtime();

arangod/Graph/Providers/ProviderTracer.h

@@ -94,6 +94,7 @@ class ProviderTracer {
   aql::TraversalStats stealStats();
 
   [[nodiscard]] transaction::Methods* trx();
+  [[nodiscard]] TRI_vocbase_t const& vocbase() const;
 
   void prepareContext(aql::InputAqlItemRow input);
   void unPrepareContext();

arangod/Graph/Providers/SingleServerProvider.cpp

@@ -261,6 +261,14 @@ arangodb::transaction::Methods* SingleServerProvider<Step>::trx() {
   return _trx.get();
 }
 
+template<class Step>
+TRI_vocbase_t const& SingleServerProvider<Step>::vocbase() const {
+  TRI_ASSERT(_trx != nullptr);
+  TRI_ASSERT(_trx->state() != nullptr);
+  TRI_ASSERT(_trx->transactionContextPtr() != nullptr);
+  return _trx.get()->vocbase();
+}
+
 template<class Step>
 arangodb::aql::TraversalStats SingleServerProvider<Step>::stealStats() {
   auto t = _stats;

arangod/Graph/Providers/SingleServerProvider.h

@@ -115,6 +115,7 @@ class SingleServerProvider {
   void destroyEngines(){};
 
   [[nodiscard]] transaction::Methods* trx();
+  [[nodiscard]] TRI_vocbase_t const& vocbase() const;
 
   aql::TraversalStats stealStats();
 

arangod/Sharding/ShardingFeature.cpp

@@ -79,6 +79,12 @@ void ShardingFeature::prepare() {
         return std::make_unique<ShardingStrategyEnterpriseHashSmartEdge>(
             sharding);
       });
+  registerFactory(
+      ShardingStrategyEnterpriseHexSmartVertex::NAME,
+      [](ShardingInfo* sharding) {
+        return std::make_unique<ShardingStrategyEnterpriseHexSmartVertex>(
+            sharding);
+      });
 #else
   // in the Community Edition register some stand-ins for the sharding
   // strategies only available in the Enterprise Edition
@@ -186,9 +192,20 @@ std::string ShardingFeature::getDefaultShardingStrategyForNewCollection(
   bool isEdge =
       TRI_COL_TYPE_EDGE == VelocyPackHelper::getNumericValue<uint32_t>(
                                properties, "type", TRI_COL_TYPE_DOCUMENT);
-  if (isSmart && isEdge) {
-    // smart edge collection
-    return ShardingStrategyEnterpriseHashSmartEdge::NAME;
+  if (isSmart) {
+    if (isEdge) {
+      // smart edge collection
+      return ShardingStrategyEnterpriseHashSmartEdge::NAME;
+    } else {
+      VPackSlice sga = properties.get(StaticStrings::GraphSmartGraphAttribute);
+      if (sga.isNone()) {
+        // In case we do have a SmartVertex collection without a
+        // SmartGraphAttribute given, we use a different sharding strategy.
+        // In case it is given, we fall back to the default ShardingStrategyHash
+        // strategy.
+        return ShardingStrategyEnterpriseHexSmartVertex::NAME;
+      }
+    }
   }
 #endif
 

arangod/Sharding/ShardingInfo.cpp

@@ -367,10 +367,11 @@ void ShardingInfo::distributeShardsLike(std::string const& cid,
   }
 
   if (!usesSameShardingStrategy(other)) {
-    // other collection has a different sharding strategy
-    // adjust our sharding so it uses the same strategy as the other collection
     auto& server = _collection->vocbase().server();
     auto& shr = server.getFeature<ShardingFeature>();
+    // other collection has a different sharding strategy
+    // adjust our sharding so it uses the same strategy as the other
+    // collection
     _shardingStrategy = shr.create(other->shardingStrategyName(), this);
   }
 

arangod/Sharding/ShardingStrategyDefault.cpp

@@ -29,11 +29,14 @@
 #include "Basics/StaticStrings.h"
 #include "Basics/hashes.h"
 #include "Cluster/ClusterFeature.h"
-#include "Cluster/ClusterInfo.h"
 #include "Cluster/ServerState.h"
 #include "Sharding/ShardingInfo.h"
 #include "VocBase/LogicalCollection.h"
 
+#ifdef USE_ENTERPRISE
+#include "Enterprise/Sharding/ShardingStrategyEE.h"
+#endif
+
 #include <velocypack/Builder.h>
 #include <velocypack/Slice.h>
 
@@ -415,3 +418,15 @@ ShardingStrategyHash::ShardingStrategyHash(ShardingInfo* sharding)
 
   ::preventUseOnSmartEdgeCollection(_sharding->collection(), NAME);
 }
+
+bool ShardingStrategyHash::isCompatible(const ShardingStrategy* other) const {
+#ifdef USE_ENTERPRISE
+  return (name() == other->name() ||
+          other->name() == ShardingStrategyEnterpriseHexSmartVertex::NAME) &&
+         (_sharding->shardKeys().size() == 1 &&
+          (_sharding->shardKeys().at(0) == StaticStrings::PrefixOfKeyString ||
+           _sharding->shardKeys().at(0) == StaticStrings::PostfixOfKeyString));
+#else
+  return name() == other->name();
+#endif
+}

arangod/Sharding/ShardingStrategyDefault.h

@@ -96,15 +96,13 @@ class ShardingStrategyHashBase : public ShardingStrategy {
                                     bool docComplete, ErrorCode& error,
                                     std::string_view const& key);
 
- private:
-  void determineShards();
-
  protected:
   ShardingInfo* _sharding;
   std::vector<ShardID> _shards;
   bool _usesDefaultShardKeys;
   std::atomic<bool> _shardsSet;
   Mutex _shardsSetMutex;
+  void determineShards();
 };
 
 /// @brief old version of the sharding used in the Community Edition
@@ -155,6 +153,8 @@ class ShardingStrategyHash final : public ShardingStrategyHashBase {
   std::string const& name() const override { return NAME; }
 
   static std::string const NAME;
+
+  bool isCompatible(ShardingStrategy const* other) const override;
 };
 
 }  // namespace arangodb

arangod/VocBase/LogicalCollection.cpp

@@ -51,6 +51,10 @@
 #include "VocBase/ManagedDocumentResult.h"
 #include "VocBase/Validators.h"
 
+#ifdef USE_ENTERPRISE
+#include "Enterprise/Sharding/ShardingStrategyEE.h"
+#endif
+
 #include <velocypack/Collection.h>
 #include <velocypack/Utf8Helper.h>
 
@@ -397,6 +401,21 @@ bool LogicalCollection::hasClusterWideUniqueRevs() const noexcept {
 
 bool LogicalCollection::mustCreateKeyOnCoordinator() const noexcept {
   TRI_ASSERT(ServerState::instance()->isRunningInCluster());
+
+#ifdef USE_ENTERPRISE
+  if (_sharding->shardingStrategyName() ==
+      ShardingStrategyEnterpriseHexSmartVertex::NAME) {
+    TRI_ASSERT(isSmart());
+    TRI_ASSERT(type() == TRI_COL_TYPE_DOCUMENT);
+    TRI_ASSERT(!hasSmartGraphAttribute());
+    // if we've a SmartVertex collection sitting inside an EnterpriseGraph
+    // we need to have the key before we can call the "getResponsibleShards"
+    // method because without it, we cannot calculate which shard will be the
+    // responsible one.
+    return true;
+  }
+#endif
+
   // when there is more than 1 shard, or if we do have a satellite
   // collection, we need to create the key on the coordinator.
   return numberOfShards() != 1;

arangod/VocBase/LogicalCollection.h

@@ -158,6 +158,9 @@ class LogicalCollection : public LogicalDataSource {
   bool hasSmartJoinAttribute() const noexcept {
     return !_smartJoinAttribute.empty();
   }
+  bool hasSmartGraphAttribute() const noexcept {
+    return !_smartGraphAttribute.empty();
+  }
 
   bool isLocalSmartEdgeCollection() const noexcept;
   bool isRemoteSmartEdgeCollection() const noexcept;
@@ -170,6 +173,7 @@ class LogicalCollection : public LogicalDataSource {
   bool isSmart() const noexcept { return false; }
   bool isSmartChild() const noexcept { return false; }
   bool hasSmartJoinAttribute() const noexcept { return false; }
+  bool hasSmartGraphAttribute() const noexcept { return false; }
 
   bool isLocalSmartEdgeCollection() const noexcept { return false; }
   bool isRemoteSmartEdgeCollection() const noexcept { return false; }