Merge branch 'devel' of https://github.com/arangodb/arangodb into fea…

…ture/smart-join-views-2

* 'devel' of https://github.com/arangodb/arangodb:
  rawPayload shouldn't return the full reply buffer (#10319)
  micro optimizations (#10316)
  test attempt to increase max collection name length from 64 chars to 256 (#9890)
  Feature/force backup (#10265)
  upgrade vpack library (#10314)
  avoid string copies in several cases (#10317)
  Round index estimates when comparing plan, we do not really care for exact equallity, they should only not be off by much (#10312)
  make ccache optional (#10310)

3rdParty/iresearch/CMakeLists.txt

@@ -52,11 +52,15 @@ endif()
 ### setup ccache
 ################################################################################
 
-find_program(CCACHE_FOUND ccache)
+option(USE_CCACHE "Use CCACHE if present" ON)
 
-if(CCACHE_FOUND)
-  set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
-endif(CCACHE_FOUND)
+if (USE_CCACHE)
+  find_program(CCACHE_FOUND ccache)
+
+  if(CCACHE_FOUND)
+    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
+  endif(CCACHE_FOUND)
+endif()
 
 if (USE_OPTIMIZE_FOR_ARCHITECTURE)
   include(OptimizeForArchitecture)

3rdParty/velocypack/include/velocypack/Buffer.h

@@ -55,11 +55,12 @@ class Buffer {
 
   Buffer(Buffer const& that) : Buffer() {
     if (that._size > 0) {
-      if (that._size > sizeof(_local)) {
-        _buffer = static_cast<T*>(malloc(checkOverflow(sizeof(T) * that._size)));
+      if (that._size > sizeof(that._local)) {
+        _buffer = static_cast<T*>(malloc(checkOverflow(that._size)));
         ensureValidPointer(_buffer);
         _capacity = that._size;
       } else {
+        VELOCYPACK_ASSERT(_buffer == &_local[0]);
         _capacity = sizeof(_local);
       }
       memcpy(_buffer, that._buffer, checkOverflow(that._size));
@@ -73,10 +74,9 @@ class Buffer {
         // our own buffer is big enough to hold the data
         initWithNone();
         memcpy(_buffer, that._buffer, checkOverflow(that._size));
-      }
-      else {
+      } else {
         // our own buffer is not big enough to hold the data
-        T* buffer = static_cast<T*>(malloc(checkOverflow(sizeof(T) * that._size)));
+        T* buffer = static_cast<T*>(malloc(checkOverflow(that._size)));
         ensureValidPointer(buffer);
         buffer[0] = '\x00';
         memcpy(buffer, that._buffer, checkOverflow(that._size));
@@ -94,8 +94,11 @@ class Buffer {
   }
 
   Buffer(Buffer&& that) noexcept : _buffer(_local), _capacity(sizeof(_local)) {
+    poison(_buffer, _capacity);
+    initWithNone();
     if (that._buffer == that._local) {
-      memcpy(_buffer, that._buffer, static_cast<std::size_t>(that._size));
+      VELOCYPACK_ASSERT(that._capacity == sizeof(that._local));
+      memcpy(_buffer, that._buffer, checkOverflow(that._size));
     } else {
       _buffer = that._buffer;
       _capacity = that._capacity;
@@ -104,23 +107,28 @@ class Buffer {
     }
     _size = that._size;
     that._size = 0;
+    that.initWithNone();
   }
 
   Buffer& operator=(Buffer&& that) noexcept {
     if (this != &that) {
+      if (_buffer != _local) {
+        free(_buffer);
+      }
       if (that._buffer == that._local) {
-        memcpy(_buffer, that._buffer, static_cast<std::size_t>(that._size));
+        _buffer = _local;
+        _capacity = sizeof(_local);
+        initWithNone();
+        memcpy(_buffer, that._buffer, checkOverflow(that._size));
       } else {
-        if (_buffer != _local) {
-          free(_buffer);
-        }
         _buffer = that._buffer;
         _capacity = that._capacity;
         that._buffer = that._local;
         that._capacity = sizeof(that._local);
       }
       _size = that._size;
       that._size = 0;
+      that.initWithNone();
     }
     return *this;
   }
@@ -281,11 +289,11 @@ class Buffer {
     VELOCYPACK_ASSERT(newLen > 0);
     T* p;
     if (_buffer != _local) {
-      p = static_cast<T*>(realloc(_buffer, checkOverflow(sizeof(T) * newLen)));
+      p = static_cast<T*>(realloc(_buffer, checkOverflow(newLen)));
       ensureValidPointer(p);
       // realloc will have copied the old data
     } else {
-      p = static_cast<T*>(malloc(checkOverflow(sizeof(T) * newLen)));
+      p = static_cast<T*>(malloc(checkOverflow(newLen)));
       ensureValidPointer(p);
       // copy existing data into buffer
       memcpy(p, _buffer, checkOverflow(_size));

3rdParty/velocypack/include/velocypack/Builder.h

@@ -91,37 +91,58 @@ class Builder {
  public:
   Options const* options;
 
-  // create an empty Builder, using default Options
-  Builder();
-
-  explicit Builder(Options const* options);
+  // create an empty Builder, using Options
+  explicit Builder(Options const* options = &Options::Defaults);
+  
+  // create an empty Builder, using an existing buffer
   explicit Builder(std::shared_ptr<Buffer<uint8_t>> const& buffer,
                    Options const* options = &Options::Defaults);
+
+  // create a Builder that uses an existing Buffer. the Builder will not
+  // claim ownership for this Buffer
   explicit Builder(Buffer<uint8_t>& buffer,
                    Options const* options = &Options::Defaults);
+
+  // populate a Builder from a Slice
   explicit Builder(Slice slice, Options const* options = &Options::Defaults);
+
   ~Builder() = default;
 
   Builder(Builder const& that);
   Builder& operator=(Builder const& that);
-  Builder(Builder&& that);
-  Builder& operator=(Builder&& that);
+  Builder(Builder&& that) noexcept;
+  Builder& operator=(Builder&& that) noexcept;
 
-  // get a const reference to the Builder's Buffer object
-  std::shared_ptr<Buffer<uint8_t>> const& buffer() const { return _buffer; }
+  // get a reference to the Builder's Buffer object
+  // note: this object may be a nullptr if the buffer was already stolen
+  // from the Builder, or if the Builder has no ownership for the Buffer
+  std::shared_ptr<Buffer<uint8_t>> const& buffer() const { 
+    return _buffer; 
+  }
+
+  Buffer<uint8_t>& bufferRef() const { 
+    if (_bufferPtr == nullptr) {
+      throw Exception(Exception::InternalError, "Builder has no Buffer");
+    }
+    return *_bufferPtr; 
+  }
 
   // steal the Builder's Buffer object. afterwards the Builder
-  // is unusable
+  // is unusable - note: this may return a nullptr if the Builder does not
+  // own the Buffer!
   std::shared_ptr<Buffer<uint8_t>> steal() {
     // After a steal the Builder is broken!
-    std::shared_ptr<Buffer<uint8_t>> res = _buffer;
-    _buffer.reset();
+    std::shared_ptr<Buffer<uint8_t>> res(std::move(_buffer));
     _bufferPtr = nullptr;
-    _pos = 0;
+    _start = nullptr;
+    clear();
     return res;
   }
 
-  uint8_t const* data() const noexcept { return _bufferPtr->data(); }
+  uint8_t const* data() const noexcept {
+    VELOCYPACK_ASSERT(_bufferPtr != nullptr);
+    return _bufferPtr->data(); 
+  }
 
   std::string toString() const;
 
@@ -153,6 +174,7 @@ class Builder {
     (void) checkOverflow(_pos + len);
 #endif
 
+    VELOCYPACK_ASSERT(_bufferPtr != nullptr);
     _bufferPtr->reserve(len);
     _start = _bufferPtr->data();
   }
@@ -161,8 +183,11 @@ class Builder {
   void clear() noexcept {
     _pos = 0;
     _stack.clear();
-    VELOCYPACK_ASSERT(_bufferPtr != nullptr);
-    _bufferPtr->reset();
+    _index.clear();
+    if (_bufferPtr != nullptr) {
+      _bufferPtr->reset();
+      _start = _bufferPtr->data();
+    }
     _keyWritten = false;
   }
 
@@ -564,8 +589,8 @@ class Builder {
   uint8_t* addInternal(char const* attrName, std::size_t attrLength, T const& sub) {
     bool haveReported = false;
     if (!_stack.empty()) {
-      ValueLength& tos = _stack.back();
-      if (VELOCYPACK_UNLIKELY(_start[tos] != 0x0b && _start[tos] != 0x14)) {
+      ValueLength const to = _stack.back();
+      if (VELOCYPACK_UNLIKELY(_start[to] != 0x0b && _start[to] != 0x14)) {
         throw Exception(Exception::BuilderNeedOpenObject);
       }
       if (VELOCYPACK_UNLIKELY(_keyWritten)) {
@@ -621,9 +646,9 @@ class Builder {
   void openCompoundValue(uint8_t type) {
     bool haveReported = false;
     if (!_stack.empty()) {
-      ValueLength& tos = _stack.back();
+      ValueLength const to = _stack.back();
       if (!_keyWritten) {
-        if (VELOCYPACK_UNLIKELY(_start[tos] != 0x06 && _start[tos] != 0x13)) {
+        if (VELOCYPACK_UNLIKELY(_start[to] != 0x06 && _start[to] != 0x13)) {
           throw Exception(Exception::BuilderNeedOpenArray);
         }
         reportAdd();
@@ -797,11 +822,13 @@ struct ObjectBuilder final : public BuilderContainer,
   }
   ~ObjectBuilder() {
     try {
-      builder->close();
+      if (!builder->isClosed()) {
+        builder->close();
+      }
     } catch (...) {
       // destructors must not throw. however, we can at least
       // signal something is very wrong in debug mode
-      VELOCYPACK_ASSERT(false);
+      VELOCYPACK_ASSERT(builder->isClosed());
     }
   }
 };
@@ -826,11 +853,13 @@ struct ArrayBuilder final : public BuilderContainer,
   }
   ~ArrayBuilder() {
     try {
-      builder->close();
+      if (!builder->isClosed()) {
+        builder->close();
+      }
     } catch (...) {
       // destructors must not throw. however, we can at least
       // signal something is very wrong in debug mode
-      VELOCYPACK_ASSERT(false);
+      VELOCYPACK_ASSERT(builder->isClosed());
     }
   }
 };

3rdParty/velocypack/include/velocypack/HexDump.h

@@ -41,21 +41,23 @@ struct HexDump {
   HexDump() = delete;
 
   HexDump(Slice const& slice, int valuesPerLine = 16,
-          std::string const& separator = " ")
-      : slice(slice), valuesPerLine(valuesPerLine), separator(separator) {}
+          std::string const& separator = " ", std::string const& header = "0x")
+      : slice(slice), valuesPerLine(valuesPerLine), separator(separator), header(header) {}
 
   HexDump(Slice const* slice, int valuesPerLine = 16,
-          std::string const& separator = " ")
-      : HexDump(*slice, valuesPerLine, separator) {}
+          std::string const& separator = " ", std::string const& header = "0x")
+      : HexDump(*slice, valuesPerLine, separator, header) {}
 
-  static std::string toHex(uint8_t value);
+  static std::string toHex(uint8_t value, std::string const& header = "0x");
+  static void appendHex(std::string& result, uint8_t value);
   std::string toString() const;
 
   friend std::ostream& operator<<(std::ostream&, HexDump const&);
 
   Slice const slice;
   int valuesPerLine;
   std::string separator;
+  std::string header;
 };
 
 }  // namespace arangodb::velocypack

3rdParty/velocypack/include/velocypack/Options.h

@@ -39,24 +39,48 @@ struct Options;
 class Slice;
 
 struct CustomTypeHandler {
-  virtual ~CustomTypeHandler() {}
+  virtual ~CustomTypeHandler() = default;
   virtual void dump(Slice const&, Dumper*, Slice const&);
   virtual std::string toString(Slice const&, Options const*, Slice const&);
 };
 
 struct Options {
+  // Behavior to be applied when dumping VelocyPack values that cannot be
+  // expressed in JSON without data loss
   enum UnsupportedTypeBehavior {
+    // convert any non-JSON-representable value to null
     NullifyUnsupportedType,
+    // emit a JSON string "(non-representable type ...)"
     ConvertUnsupportedType,
+    // throw an exception for any non-JSON-representable value
     FailOnUnsupportedType
   };
 
+  // Behavior to be applied when building VelocyPack Array/Object values
+  // with a Builder
+  enum PaddingBehavior {
+    // use padding - fill unused head bytes with zero-bytes (ASCII NUL) in
+    // order to avoid a later memmove
+    UsePadding,
+    // don't pad and do not fill any gaps with zero-bytes (ASCII NUL). 
+    // instead, memmove data down so there is no gap between the head bytes
+    // and the payload
+    NoPadding,
+    // pad in cases the Builder considers it useful, and don't pad in other
+    // cases when the Builder doesn't consider it useful
+    Flexible
+  };
+
   Options() {}
 
   // Dumper behavior when a VPack value is serialized to JSON that
   // has no JSON equivalent
   UnsupportedTypeBehavior unsupportedTypeBehavior = FailOnUnsupportedType;
+
+  // Builder behavior w.r.t. padding or memmoving data
+  PaddingBehavior paddingBehavior = PaddingBehavior::Flexible;
 
+  // custom attribute translator for integer keys
   AttributeTranslator* attributeTranslator = nullptr;
 
   // custom type handler used for processing custom types by Dumper and Slicer

3rdParty/velocypack/include/velocypack/Sink.h

@@ -42,7 +42,7 @@ struct Sink {
   Sink(Sink const&) = delete;
   Sink& operator=(Sink const&) = delete;
 
-  virtual ~Sink() {}
+  virtual ~Sink() = default;
   virtual void push_back(char c) = 0;
   virtual void append(std::string const& p) = 0;
   virtual void append(char const* p) = 0;

3rdParty/velocypack/include/velocypack/Slice.h

@@ -47,13 +47,12 @@
 namespace arangodb {
 namespace velocypack {
 
-class Slice {
-  // This class provides read only access to a VPack value, it is
-  // intentionally light-weight (only one pointer value), such that
-  // it can easily be used to traverse larger VPack values.
-
-  // A Slice does not own the VPack data it points to!
+// This class provides read only access to a VPack value, it is
+// intentionally light-weight (only one pointer value), such that
+// it can easily be used to traverse larger VPack values.
 
+// A Slice does not own the VPack data it points to!
+class Slice {
   friend class Builder;
   friend class ArrayIterator;
   friend class ObjectIterator;
@@ -1105,6 +1104,9 @@ class Slice {
   }
 };
 
+static_assert(!std::is_polymorphic<Slice>::value, "Slice must not be polymorphic");
+static_assert(!std::has_virtual_destructor<Slice>::value, "Slice must not have virtual dtor");
+
 }  // namespace arangodb::velocypack
 }  // namespace arangodb
 

3rdParty/velocypack/include/velocypack/velocypack-aliases.h

@@ -70,6 +70,7 @@ using VPackNormalizedCompare = arangodb::velocypack::NormalizedCompare;
 #ifdef VELOCYPACK_BUFFER_H
 #ifndef VELOCYPACK_ALIAS_BUFFER
 #define VELOCYPACK_ALIAS_BUFFER
+using VPackCharBuffer = arangodb::velocypack::CharBuffer;
 using VPackBufferUInt8 = arangodb::velocypack::UInt8Buffer;
 template<typename T> using VPackBuffer = arangodb::velocypack::Buffer<T>;
 #endif