Optimize IterKey

db/dbformat.cc

@@ -201,6 +201,32 @@ LookupKey::LookupKey(const Slice& _user_key, SequenceNumber s,
   end_ = dst;
 }
 
+void IterKey::TrimAppend(const size_t shared_len, const char* non_shared_data,
+                         const size_t non_shared_len) {
+  assert(shared_len <= key_size_);
+  size_t total_size = shared_len + non_shared_len;
+
+  if (IsKeyPinned() /* key is not in buf_ */) {
+    // Copy the key from external memory to buf_ (copy shared_len bytes)
+    EnlargeBufferIfNeeded(total_size);
+    memcpy(buf(), key_, shared_len);
+  } else if (total_size > buf_size_) {
+    // Need to allocate space, delete previous space
+    char* p = new char[total_size];
+    memcpy(p, key_, shared_len);
+
+    if (buf_size_ != sizeof(space_)) {
+      delete[] buf_;
+    }
+
+    buf_ = p;
+    buf_size_ = total_size;
+  }
+  memcpy(buf() + shared_len, non_shared_data, non_shared_len);
+  key_ = buf();
+  key_size_ = total_size;
+}
+
 void IterKey::EnlargeBuffer(size_t key_size) {
   // If size is smaller than buffer size, continue using current buffer,
   // or the static allocated one, as default

db/dbformat.h

@@ -423,6 +423,18 @@ inline uint64_t GetInternalKeySeqno(const Slice& internal_key) {
   return num >> 8;
 }
 
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Warray-bounds"
+#pragma GCC diagnostic ignored "-Wstringop-overflow"
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+#elif defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunknown-warning-option"
+#pragma clang diagnostic ignored "-Warray-bounds"
+#pragma clang diagnostic ignored "-Wstringop-overflow"
+#pragma clang diagnostic ignored "-Wmaybe-uninitialized"
+#endif
 // The class to store keys in an efficient way. It allows:
 // 1. Users can either copy the key into it, or have it point to an unowned
 //    address.
@@ -432,8 +444,7 @@ inline uint64_t GetInternalKeySeqno(const Slice& internal_key) {
 class IterKey {
  public:
   IterKey()
-      : buf_(space_),
-        key_(buf_),
+      : key_(space_),
         key_size_(0),
         buf_size_(sizeof(space_)),
         is_user_key_(true) {}
@@ -476,31 +487,7 @@ class IterKey {
   // shared_len: bytes in [0, shard_len-1] would be remained
   // non_shared_data: data to be append, its length must be >= non_shared_len
   void TrimAppend(const size_t shared_len, const char* non_shared_data,
-                  const size_t non_shared_len) {
-    assert(shared_len <= key_size_);
-    size_t total_size = shared_len + non_shared_len;
-
-    if (IsKeyPinned() /* key is not in buf_ */) {
-      // Copy the key from external memory to buf_ (copy shared_len bytes)
-      EnlargeBufferIfNeeded(total_size);
-      memcpy(buf_, key_, shared_len);
-    } else if (total_size > buf_size_) {
-      // Need to allocate space, delete previous space
-      char* p = new char[total_size];
-      memcpy(p, key_, shared_len);
-
-      if (buf_ != space_) {
-        delete[] buf_;
-      }
-
-      buf_ = p;
-      buf_size_ = total_size;
-    }
-
-    memcpy(buf_ + shared_len, non_shared_data, non_shared_len);
-    key_ = buf_;
-    key_size_ = total_size;
-  }
+                  const size_t non_shared_len);
 
   Slice SetKey(const Slice& key, bool copy = true) {
     // is_user_key_ expected to be set already via SetIsUserKey
@@ -535,8 +522,8 @@ class IterKey {
     assert(IsKeyPinned() == true);
 
     Reserve(key_size_);
-    memcpy(buf_, key_, key_size_);
-    key_ = buf_;
+    memcpy(buf(), key_, key_size_);
+    key_ = buf();
   }
 
   // Update the sequence number in the internal key.  Guarantees not to
@@ -546,14 +533,14 @@ class IterKey {
     assert(key_size_ >= kNumInternalBytes);
     if (ts) {
       assert(key_size_ >= kNumInternalBytes + ts->size());
-      memcpy(&buf_[key_size_ - kNumInternalBytes - ts->size()], ts->data(),
+      memcpy(&buf()[key_size_ - kNumInternalBytes - ts->size()], ts->data(),
              ts->size());
     }
     uint64_t newval = (seq << 8) | t;
-    EncodeFixed64(&buf_[key_size_ - kNumInternalBytes], newval);
+    EncodeFixed64(&buf()[key_size_ - kNumInternalBytes], newval);
   }
 
-  bool IsKeyPinned() const { return (key_ != buf_); }
+  bool IsKeyPinned() const { return (key_ != buf()); }
 
   // If `ts` is provided, user_key should not contain timestamp,
   // and `ts` is appended after user_key.
@@ -567,16 +554,16 @@ class IterKey {
     size_t ts_sz = (ts != nullptr ? ts->size() : 0);
     EnlargeBufferIfNeeded(psize + usize + sizeof(uint64_t) + ts_sz);
     if (psize > 0) {
-      memcpy(buf_, key_prefix.data(), psize);
+      memcpy(buf(), key_prefix.data(), psize);
     }
-    memcpy(buf_ + psize, user_key.data(), usize);
+    memcpy(buf() + psize, user_key.data(), usize);
     if (ts) {
-      memcpy(buf_ + psize + usize, ts->data(), ts_sz);
+      memcpy(buf() + psize + usize, ts->data(), ts_sz);
     }
-    EncodeFixed64(buf_ + usize + psize + ts_sz,
+    EncodeFixed64(buf() + usize + psize + ts_sz,
                   PackSequenceAndType(s, value_type));
 
-    key_ = buf_;
+    key_ = buf();
     key_size_ = psize + usize + sizeof(uint64_t) + ts_sz;
     is_user_key_ = false;
   }
@@ -605,29 +592,36 @@ class IterKey {
   void EncodeLengthPrefixedKey(const Slice& key) {
     auto size = key.size();
     EnlargeBufferIfNeeded(size + static_cast<size_t>(VarintLength(size)));
-    char* ptr = EncodeVarint32(buf_, static_cast<uint32_t>(size));
+    char* ptr = EncodeVarint32(buf(), static_cast<uint32_t>(size));
     memcpy(ptr, key.data(), size);
-    key_ = buf_;
+    key_ = buf();
     is_user_key_ = true;
   }
 
   bool IsUserKey() const { return is_user_key_; }
 
  private:
-  char* buf_;
   const char* key_;
-  size_t key_size_;
-  size_t buf_size_;
-  char space_[32];  // Avoid allocation for short keys
-  bool is_user_key_;
+  size_t key_size_ : 32;
+  size_t buf_size_ : 31;
+  size_t is_user_key_ : 1;
+  union {
+    char* buf_;
+    char space_[48];  // Avoid allocation for short keys
+  };
+
+  char* buf() { return buf_size_ <= sizeof(space_) ? space_ : buf_; }
+  const char* buf() const {
+    return buf_size_ <= sizeof(space_) ? space_ : buf_;
+  }
 
   Slice SetKeyImpl(const Slice& key, bool copy) {
     size_t size = key.size();
     if (copy) {
       // Copy key to buf_
       EnlargeBufferIfNeeded(size);
-      memcpy(buf_, key.data(), size);
-      key_ = buf_;
+      memcpy(buf(), key.data(), size);
+      key_ = buf();
     } else {
       // Update key_ to point to external memory
       key_ = key.data();
@@ -637,9 +631,8 @@ class IterKey {
   }
 
   void ResetBuffer() {
-    if (buf_ != space_) {
+    if (sizeof(space_) != buf_size_) {
       delete[] buf_;
-      buf_ = space_;
     }
     buf_size_ = sizeof(space_);
     key_size_ = 0;
@@ -660,6 +653,11 @@ class IterKey {
 
   void EnlargeBuffer(size_t key_size);
 };
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#elif defined(__clang__)
+#pragma clang diagnostic pop
+#endif
 
 // Convert from a SliceTransform of user keys, to a SliceTransform of
 // internal keys.