APM-237: computed values (#15589)

Co-authored-by: cpjulia <julia@arangodb.com>

arangod/Aql/AqlFunctionsInternalCache.cpp

@@ -93,7 +93,7 @@ icu::RegexMatcher* AqlFunctionsInternalCache::buildSplitMatcher(
 }
 
 arangodb::ValidatorBase* AqlFunctionsInternalCache::buildValidator(
-    VPackSlice const& validatorParams) {
+    VPackSlice validatorParams) {
   auto matcherIter =
       _validatorCache
           .try_emplace(

arangod/Aql/AqlFunctionsInternalCache.h

@@ -64,8 +64,7 @@ class AqlFunctionsInternalCache final {
   /// validation.
   //                              But it is able to handle other validation
   //                              types without change.
-  arangodb::ValidatorBase* buildValidator(
-      VPackSlice const& validatorDescription);
+  arangodb::ValidatorBase* buildValidator(VPackSlice validatorDescription);
 
   /// @brief inspect a LIKE pattern from a string, and remove all
   /// of its escape characters. will stop at the first wildcards found.
@@ -101,8 +100,6 @@ class AqlFunctionsInternalCache final {
       _likeCache;
   /// @brief cache for validators -- This is currently only used for JSONSchema
   /// validation.
-  //                                 But it is able to handle other validation
-  //                                 types without change.
   std::unordered_map<std::size_t, std::unique_ptr<arangodb::ValidatorBase>>
       _validatorCache;
   /// @brief a reusable string object for pattern generation

arangod/Aql/Ast.cpp

@@ -2358,7 +2358,8 @@ size_t Ast::extractParallelism(AstNode const* optionsNode) {
 /// this does not only optimize but also performs a few validations after
 /// bind parameter injection. merging this pass with the regular AST
 /// optimizations saves one extra pass over the AST
-void Ast::validateAndOptimize(transaction::Methods& trx) {
+void Ast::validateAndOptimize(transaction::Methods& trx,
+                              Ast::ValidateAndOptimizeOptions const& options) {
   ::ValidateAndOptimizeContext context(trx);
 
   auto preVisitor = [&](AstNode const* node) -> bool {
@@ -2570,8 +2571,8 @@ void Ast::validateAndOptimize(transaction::Methods& trx) {
 
       if (ctx->stopOptimizationRequests == 0) {
         // optimization allowed
-        return this->optimizeFunctionCall(ctx->trx,
-                                          ctx->aqlFunctionsInternalCache, node);
+        return this->optimizeFunctionCall(
+            ctx->trx, ctx->aqlFunctionsInternalCache, node, options);
       }
       // optimization not allowed
       return node;
@@ -3691,7 +3692,8 @@ AstNode* Ast::optimizeAttributeAccess(
 /// @brief optimizes a call to a built-in function
 AstNode* Ast::optimizeFunctionCall(
     transaction::Methods& trx,
-    AqlFunctionsInternalCache& aqlFunctionsInternalCache, AstNode* node) {
+    AqlFunctionsInternalCache& aqlFunctionsInternalCache, AstNode* node,
+    Ast::ValidateAndOptimizeOptions const& options) {
   TRI_ASSERT(node != nullptr);
   TRI_ASSERT(node->type == NODE_TYPE_FCALL);
   TRI_ASSERT(node->numMembers() == 1);
@@ -3798,6 +3800,12 @@ AstNode* Ast::optimizeFunctionCall(
     return node;
   }
 
+  if (!options.optimizeNonCacheable &&
+      !func->hasFlag(Function::Flags::Cacheable)) {
+    // non-cacheable function
+    return node;
+  }
+
   if (!node->getMember(0)->isConstant()) {
     // arguments to function call are not constant
     return node;

arangod/Aql/Ast.h

@@ -87,6 +87,13 @@ class Ast {
   /// @brief destroy the AST
   ~Ast();
 
+  struct ValidateAndOptimizeOptions {
+    // try to optimize non-cacheable expressions as well. this is true for
+    // most use cases, but we cannot activate this for computed values, as
+    // we need a freshly calculated value every time.
+    bool optimizeNonCacheable = true;
+  };
+
   // frees all data
   void clear() noexcept;
 
@@ -451,7 +458,8 @@ class Ast {
   static size_t extractParallelism(AstNode const* optionsNode);
 
   /// @brief optimizes the AST
-  void validateAndOptimize(transaction::Methods&);
+  void validateAndOptimize(transaction::Methods&,
+                           ValidateAndOptimizeOptions const& options);
 
   /// @brief determines the variables referenced in an expression
   static void getReferencedVariables(AstNode const*, VarSet&);
@@ -551,7 +559,8 @@ class Ast {
 
   /// @brief optimizes a call to a built-in function
   AstNode* optimizeFunctionCall(transaction::Methods&,
-                                AqlFunctionsInternalCache&, AstNode*);
+                                AqlFunctionsInternalCache&, AstNode*,
+                                ValidateAndOptimizeOptions const& options);
 
   /// @brief optimizes indexed access, e.g. a[0] or a['foo']
   AstNode* optimizeIndexedAccess(AstNode*);

arangod/Aql/Expression.cpp

@@ -48,6 +48,7 @@
 #include "V8/v8-globals.h"
 #include "V8/v8-vpack.h"
 
+#include <velocypack/Buffer.h>
 #include <velocypack/Builder.h>
 #include <velocypack/Iterator.h>
 #include <velocypack/Sink.h>
@@ -91,7 +92,7 @@ void Expression::variables(VarSet& result) const {
 /// @brief execute the expression
 AqlValue Expression::execute(ExpressionContext* ctx, bool& mustDestroy) {
   TRI_ASSERT(ctx != nullptr);
-  prepareForExecution(*ctx);
+  prepareForExecution();
 
   TRI_ASSERT(_type != UNPROCESSED);
 
@@ -164,7 +165,7 @@ void Expression::replaceAttributeAccess(
 void Expression::freeInternals() noexcept {
   switch (_type) {
     case JSON:
-      delete[] _data;
+      velocypack_free(_data);
       _data = nullptr;
       break;
 
@@ -336,7 +337,7 @@ void Expression::determineType() {
   }
 }
 
-void Expression::initAccessor(ExpressionContext& ctx) {
+void Expression::initAccessor() {
   TRI_ASSERT(_type == ATTRIBUTE_ACCESS);
   TRI_ASSERT(_accessor == nullptr);
 
@@ -357,30 +358,35 @@ void Expression::initAccessor(ExpressionContext& ctx) {
   TRI_ASSERT(_accessor != nullptr);
 }
 
-/// @brief prepare the expression for execution
-void Expression::prepareForExecution(ExpressionContext& ctx) {
+/// @brief prepare the expression for execution, without an
+/// ExpressionContext.
+void Expression::prepareForExecution() {
   TRI_ASSERT(_type != UNPROCESSED);
 
-  switch (_type) {
-    case JSON: {
+  if (_type == JSON && _data == nullptr) {
+    // generate a constant value, using an on-stack Builder
+    velocypack::Buffer<uint8_t> buffer;
+    velocypack::Builder builder(buffer);
+    _node->toVelocyPackValue(builder);
+
+    if (buffer.usesLocalMemory()) {
+      // Buffer has data in its local memory. because we
+      // don't want to keep the whole Buffer object, we allocate
+      // the required space ourselves and copy things over.
+      _data = static_cast<uint8_t*>(
+          velocypack_malloc(static_cast<size_t>(buffer.size())));
       if (_data == nullptr) {
-        // generate a constant value
-        transaction::BuilderLeaser builder(&ctx.trx());
-        _node->toVelocyPackValue(*builder.get());
-
-        _data = new uint8_t[static_cast<size_t>(builder->size())];
-        memcpy(_data, builder->data(), static_cast<size_t>(builder->size()));
-      }
-      break;
-    }
-    case ATTRIBUTE_ACCESS: {
-      if (_accessor == nullptr) {
-        initAccessor(ctx);
+        // malloc returned a nullptr
+        throw std::bad_alloc();
       }
-      break;
-    }
-    default: {
+      memcpy(_data, buffer.data(), static_cast<size_t>(buffer.size()));
+    } else {
+      // we can simply steal the data from the Buffer. we
+      // own the data now.
+      _data = buffer.steal();
     }
+  } else if (_type == ATTRIBUTE_ACCESS && _accessor == nullptr) {
+    initAccessor();
   }
 }
 
@@ -1718,7 +1724,7 @@ AqlValue Expression::executeSimpleExpressionExpansion(ExpressionContext& ctx,
         if (!isBoolean) {
           AqlValue sub = executeSimpleExpression(ctx, projectionNode,
                                                  localMustDestroy, false);
-          sub.toVelocyPack(&vopts, builder, /*resolveExternals*/ false,
+          sub.toVelocyPack(&vopts, builder, /*resolveExternals*/ true,
                            /*allowUnindexed*/ false);
           if (localMustDestroy) {
             sub.destroy();

arangod/Aql/Expression.h

@@ -155,6 +155,9 @@ class Expression {
   /// changed
   void invalidateAfterReplacements();
 
+  // prepare the expression for execution
+  void prepareForExecution();
+
  private:
   // free the internal data structures
   void freeInternals() noexcept;
@@ -167,10 +170,7 @@ class Expression {
   void determineType();
 
   // init the accessor specialization
-  void initAccessor(ExpressionContext& ctx);
-
-  // prepare the expression for execution
-  void prepareForExecution(ExpressionContext& ctx);
+  void initAccessor();
 
   // execute an expression of type SIMPLE
   static AqlValue executeSimpleExpression(ExpressionContext& ctx,

arangod/Aql/Function.cpp

@@ -57,6 +57,7 @@ Function::Function(std::string const& name, char const* arguments,
       << hasFlag(Flags::CanRunOnDBServerOneShard)
       << ", canReadDocuments: " << hasFlag(Flags::CanReadDocuments)
       << ", canUseInAnalyzer: " << hasFlag(Flags::CanUseInAnalyzer)
+      << ", internal: " << hasFlag(Flags::Internal)
       << ", hasCxxImplementation: " << hasCxxImplementation()
       << ", hasConversions: " << !conversions.empty();
 
@@ -228,6 +229,8 @@ void Function::toVelocyPack(arangodb::velocypack::Builder& builder) const {
               velocypack::Value(hasFlag(Flags::CanRunOnDBServerCluster)));
   builder.add("canRunOnDBServerOneShard",
               velocypack::Value(hasFlag(Flags::CanRunOnDBServerOneShard)));
+  builder.add("canReadDocuments",
+              velocypack::Value(hasFlag(Flags::CanReadDocuments)));
   builder.add("canUseInAnalyzer",
               velocypack::Value(hasFlag(Flags::CanUseInAnalyzer)));
 

arangod/Aql/Query.cpp

@@ -396,7 +396,7 @@ std::unique_ptr<ExecutionPlan> Query::preparePlan() {
   // we have an AST, optimize the ast
   enterState(QueryExecutionState::ValueType::AST_OPTIMIZATION);
 
-  _ast->validateAndOptimize(*_trx);
+  _ast->validateAndOptimize(*_trx, {.optimizeNonCacheable = true});
 
   enterState(QueryExecutionState::ValueType::LOADING_COLLECTIONS);
 
@@ -964,7 +964,7 @@ QueryResult Query::explain() {
     }
 
     enterState(QueryExecutionState::ValueType::LOADING_COLLECTIONS);
-    _ast->validateAndOptimize(*_trx);
+    _ast->validateAndOptimize(*_trx, {.optimizeNonCacheable = true});
 
     enterState(QueryExecutionState::ValueType::PLAN_INSTANTIATION);
     std::unique_ptr<ExecutionPlan> plan =

arangod/Aql/QueryExpressionContext.h

@@ -36,7 +36,7 @@ namespace aql {
 class QueryContext;
 class AqlFunctionsInternalCache;
 
-class QueryExpressionContext : public ExpressionContext {
+class QueryExpressionContext : public aql::ExpressionContext {
  public:
   QueryExpressionContext(QueryExpressionContext const&) = delete;
   QueryExpressionContext& operator=(QueryExpressionContext const&) = delete;