Add MsgPack-based Table Slice implementation

CHANGELOG.md

@@ -11,6 +11,15 @@ Every entry has a category for which we use the following visual abbreviations:
 
 ## Unreleased
 
+- 🎁 We open-sourced our [MessagePack](http://msgpack.org)-based table slice
+  implementation, which provides a compact row-oriented encoding of data. This
+  encoding works well for binary formats (e.g., PCAP) and access patterns that
+  involve materializing entire rows. The MessagePack table slice is the new
+  default when Apache Arrow is unavailable. To enable parsing into MessagePack,
+  you can pass `--table-slice-type=msgpack` to the `import` command, or set the
+  configuration option `import.table-slice-type` to `'msgpack'`.
+  [#975](https://github.com/tenzir/vast/pull/975)
+
 - 🎁 Starting with this release, installing VAST on any Linux becomes
   significantly easier: A static binary will be provided with each release on
   the GitHub releases page. [#966](https://github.com/tenzir/vast/pull/966)

integration/default_set.yaml

@@ -140,12 +140,23 @@ tests:
     steps:
       - command: import -b zeek
         input: data/zeek/conn.log.gz
-      - command: import --table-slice-type=caf -b zeek
+      - command: import -b zeek
         input: data/zeek/dns.log.gz
       - command: export ascii 'resp_h == 192.168.1.104'
       - command: export ascii 'zeek.conn.id.resp_h == 192.168.1.104'
       - command: 'count "#timestamp >= 1970-01-01 && #type != \"vast.metrics\""'
       - command: count '#type == "zeek.conn"'
+  Table Slice Types:
+    tags: [server, import-export, table-slice-type, zeek]
+    fixture: ServerTester
+    steps:
+      - command: import -b --table-slice-type=caf zeek
+        input: data/zeek/conn.log.gz
+      - command: import -b --table-slice-type=arrow zeek
+        input: data/zeek/conn.log.gz
+      - command: import -b --table-slice-type=msgpack zeek
+        input: data/zeek/conn.log.gz
+      - command: count '#type == "zeek.conn"'
   Query Operators:
     tags: [server, operator]
     fixture: ServerTester

integration/reference/table-slice-types/step_03.ref

@@ -0,0 +1 @@
+25386

libvast/CMakeLists.txt

@@ -75,6 +75,9 @@ set(libvast_sources
     src/json.cpp
     src/logger.cpp
     src/meta_index.cpp
+    src/msgpack.cpp
+    src/msgpack_table_slice.cpp
+    src/msgpack_table_slice_builder.cpp
     src/null_bitmap.cpp
     src/operator.cpp
     src/pattern.cpp
@@ -287,6 +290,8 @@ set(tests
     test/json.cpp
     test/meta_index.cpp
     test/mmapbuf.cpp
+    test/msgpack.cpp
+    test/msgpack_table_slice.cpp
     test/offset.cpp
     test/parse_data.cpp
     test/parseable.cpp

libvast/src/msgpack.cpp

@@ -0,0 +1,140 @@
+/******************************************************************************
+ *                    _   _____   __________                                  *
+ *                   | | / / _ | / __/_  __/     Visibility                   *
+ *                   | |/ / __ |_\ \  / /          Across                     *
+ *                   |___/_/ |_/___/ /_/       Space and Time                 *
+ *                                                                            *
+ * This file is part of VAST. It is subject to the license terms in the       *
+ * LICENSE file found in the top-level directory of this distribution and at  *
+ * http://vast.io/license. No part of VAST, including this file, may be       *
+ * copied, modified, propagated, or distributed except according to the terms *
+ * contained in the LICENSE file.                                             *
+ ******************************************************************************/
+
+#include "vast/msgpack.hpp"
+
+namespace vast::msgpack {
+
+overlay array_view::data() const {
+  return overlay{data_};
+}
+
+overlay::overlay(span<const byte> buffer) : buffer_{buffer}, position_{0} {
+  // nop
+}
+
+object overlay::get() const {
+  VAST_ASSERT(position_ < buffer_.size());
+  return object{buffer_.subspan(position_)};
+}
+
+size_t overlay::next() {
+  auto fmt = static_cast<format>(buffer_[position_]);
+  auto advance = [&](size_t bytes) {
+    position_ += bytes;
+    return bytes;
+  };
+  if (is_positive_fixint(fmt) || is_negative_fixint(fmt)) {
+    return advance(1);
+  } else if (is_fixstr(fmt)) {
+    return advance(1 + fixstr_size(fmt));
+  } else if (is_fixarray(fmt)) {
+    auto n = advance(1);
+    for (size_t i = 0; i < fixarray_size(fmt); ++i)
+      n += next();
+    return n;
+  } else if (is_fixmap(fmt)) {
+    auto n = advance(1);
+    for (size_t i = 0; i < fixmap_size(fmt) * 2; ++i)
+      n += next();
+    return n;
+  }
+  switch (fmt) {
+    default:
+      break;
+    case nil:
+    case false_:
+    case true_:
+      return advance(1);
+    case uint8:
+    case int8:
+      return advance(2);
+    case uint16:
+    case int16:
+      return advance(3);
+    case uint32:
+    case int32:
+    case float32:
+      return advance(5);
+    case uint64:
+    case int64:
+    case float64:
+      return advance(9);
+    case str8:
+    case bin8:
+      return advance(1 + static_cast<uint8_t>(*at(1)));
+    case str16:
+    case bin16:
+      return advance(1 + 2 + to_num<uint16_t>(at(1)));
+    case str32:
+    case bin32:
+      return advance(1 + 4 + to_num<uint32_t>(at(1)));
+    case array16: {
+      auto size = to_num<uint16_t>(at(1));
+      auto n = advance(3);
+      for (size_t i = 0; i < size; ++i)
+        n += next();
+      return n;
+    }
+    case array32: {
+      auto size = to_num<uint32_t>(at(1));
+      auto n = advance(5);
+      for (size_t i = 0; i < size; ++i)
+        n += next();
+      return n;
+    }
+    case map16: {
+      auto size = to_num<uint16_t>(at(1));
+      auto n = advance(3);
+      for (size_t i = 0; i < size * 2; ++i)
+        n += next();
+      return n;
+    }
+    case map32: {
+      auto size = to_num<uint32_t>(at(1));
+      auto n = advance(5);
+      for (size_t i = 0; i < size * 2; ++i)
+        n += next();
+      return n;
+    }
+    case fixext1:
+      return advance(1 + 1 + 1);
+    case fixext2:
+      return advance(1 + 1 + 2);
+    case fixext4:
+      return advance(1 + 1 + 4);
+    case fixext8:
+      return advance(1 + 1 + 8);
+    case fixext16:
+      return advance(1 + 1 + 16);
+    case ext8:
+      return advance(1 + 1 + 1 + static_cast<uint8_t>(*at(1)));
+    case ext16:
+      return advance(1 + 2 + 1 + to_num<uint16_t>(at(1)));
+    case ext32:
+      return advance(1 + 4 + 1 + to_num<uint32_t>(at(1)));
+  }
+  return 0;
+}
+
+size_t overlay::next(size_t n) {
+  auto result = 0;
+  for (size_t i = 0; i < n; ++i) {
+    auto n = next();
+    VAST_ASSERT(n > 0);
+    result += n;
+  }
+  return result;
+}
+
+} // namespace vast::msgpack