Theta sketch

Makefile

@@ -1,5 +1,7 @@
 include config.mk
 
+TEST_BINARY_INPUT_PATH = test
+
 BUILDDIR := build
 TARGETDIR := lib
 
@@ -10,7 +12,7 @@ TARGET := $(TARGETDIR)/$(LIBRARY)
 INC := -I /usr/local/include
 LIB := -L /usr/local/lib -lcppunit -L lib -l$(LIB_BASE_NAME)
 
-MODULES := hll cpc kll fi
+MODULES := hll cpc kll fi theta
 
 .PHONY: all
 all: $(MODULES) $(LIBRARY)

theta/include/theta_a_not_b.hpp

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2019, Verizon Media.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+#ifndef THETA_A_NOT_B_HPP_
+#define THETA_A_NOT_B_HPP_
+
+#include <memory>
+#include <functional>
+#include <climits>
+
+#include <theta_sketch.hpp>
+
+namespace datasketches {
+
+/*
+ * author Alexander Saydakov
+ * author Lee Rhodes
+ * author Kevin Lang
+ */
+
+template<typename A>
+class theta_a_not_b_alloc {
+public:
+  explicit theta_a_not_b_alloc(uint64_t seed = update_theta_sketch_alloc<A>::builder::DEFAULT_SEED);
+
+  compact_theta_sketch_alloc<A> compute(const theta_sketch_alloc<A>& a, const theta_sketch_alloc<A>& b, bool ordered = true) const;
+
+private:
+  typedef typename std::allocator_traits<A>::template rebind_alloc<uint64_t> AllocU64;
+  uint16_t seed_hash_;
+
+};
+
+// alias with default allocator for convenience
+typedef theta_a_not_b_alloc<std::allocator<void>> theta_a_not_b;
+
+} /* namespace datasketches */
+
+#include "theta_a_not_b_impl.hpp"
+
+# endif

theta/include/theta_a_not_b_impl.hpp

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2019, Verizon Media.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+#ifndef THETA_A_NOT_B_IMPL_HPP_
+#define THETA_A_NOT_B_IMPL_HPP_
+
+#include <algorithm>
+
+namespace datasketches {
+
+/*
+ * author Alexander Saydakov
+ * author Lee Rhodes
+ * author Kevin Lang
+ */
+
+template<typename A>
+theta_a_not_b_alloc<A>::theta_a_not_b_alloc(uint64_t seed):
+seed_hash_(theta_sketch_alloc<A>::get_seed_hash(seed))
+{}
+
+constexpr uint8_t log2(uint32_t n) {
+  return (n > 1) ? 1 + log2(n >> 1) : 0;
+}
+
+constexpr uint8_t lg_size_from_count(uint32_t n, double load_factor) {
+  uint8_t lg = log2(n) + 1;
+  if (n > (1 << lg) * load_factor) lg++;
+  return lg;
+}
+
+template<typename A>
+compact_theta_sketch_alloc<A> theta_a_not_b_alloc<A>::compute(const theta_sketch_alloc<A>& a, const theta_sketch_alloc<A>& b, bool ordered) const {
+  if (a.is_empty()) return compact_theta_sketch_alloc<A>(a, ordered);
+  if (a.get_seed_hash() != seed_hash_) throw std::invalid_argument("A seed hash mismatch");
+  if (b.get_seed_hash() != seed_hash_) throw std::invalid_argument("B seed hash mismatch");
+  if (a.get_num_retained() == 0 or b.is_empty()) return compact_theta_sketch_alloc<A>(a, ordered);
+
+  const uint64_t theta = std::min(a.get_theta64(), b.get_theta64());
+  uint64_t* keys = nullptr;
+  uint32_t keys_size = 0;
+  uint32_t count = 0;
+  bool is_empty = a.is_empty();
+
+  if (b.get_num_retained() == 0) {
+    for (auto key: a) if (key < theta) ++count;
+    keys_size = count;
+    keys = AllocU64().allocate(keys_size);
+    std::copy_if(a.begin(), a.end(), keys, [theta](uint64_t key) { return key < theta; });
+    if (ordered and !a.is_ordered()) std::sort(keys, &keys[keys_size]);
+    if (count == 0 and theta == theta_sketch_alloc<A>::MAX_THETA) is_empty = true;
+    return compact_theta_sketch_alloc<A>(is_empty, theta, keys, count, seed_hash_, a.is_ordered() or ordered);
+  }
+
+  keys_size = a.get_num_retained();
+  keys = AllocU64().allocate(keys_size);
+
+  if (a.is_ordered() and b.is_ordered()) { // sort-based
+    const auto end = std::set_difference(a.begin(), a.end(), b.begin(), b.end(), keys);
+    count = end - keys;
+  } else { // hash-based
+    const uint8_t lg_size = lg_size_from_count(b.get_num_retained(), update_theta_sketch_alloc<A>::REBUILD_THRESHOLD);
+    uint64_t* b_hash_table = AllocU64().allocate(1 << lg_size);
+    std::fill(b_hash_table, &b_hash_table[1 << lg_size], 0);
+    for (auto key: b) {
+      if (key < theta) {
+        update_theta_sketch_alloc<A>::hash_search_or_insert(key, b_hash_table, lg_size);
+      } else if (b.is_ordered()) {
+        break; // early stop
+      }
+    }
+
+    // scan A lookup B
+    for (auto key: a) {
+      if (key < theta) {
+        if (!update_theta_sketch_alloc<A>::hash_search(key, b_hash_table, lg_size)) keys[count++] = key;
+      } else if (a.is_ordered()) {
+        break; // early stop
+      }
+    }
+
+    AllocU64().deallocate(b_hash_table, 1 << lg_size);
+  }
+
+  if (count == 0) {
+    AllocU64().deallocate(keys, keys_size);
+    keys = nullptr;
+    if (theta == theta_sketch_alloc<A>::MAX_THETA) is_empty = true;
+  } else if (count < keys_size) {
+    uint64_t* keys_copy = AllocU64().allocate(count);
+    std::copy(keys, &keys[count], keys_copy);
+    AllocU64().deallocate(keys, keys_size);
+    keys = keys_copy;
+    if (ordered and !a.is_ordered()) std::sort(keys, &keys[count]);
+  }
+
+  return compact_theta_sketch_alloc<A>(is_empty, theta, keys, count, seed_hash_, a.is_ordered() or ordered);
+}
+
+} /* namespace datasketches */
+
+# endif

theta/include/theta_intersection.hpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2019, Verizon Media.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+#ifndef THETA_INTERSECTION_HPP_
+#define THETA_INTERSECTION_HPP_
+
+#include <memory>
+#include <functional>
+#include <climits>
+
+#include <theta_sketch.hpp>
+
+namespace datasketches {
+
+/*
+ * author Alexander Saydakov
+ * author Lee Rhodes
+ * author Kevin Lang
+ */
+
+template<typename A>
+class theta_intersection_alloc {
+public:
+  explicit theta_intersection_alloc(uint64_t seed = update_theta_sketch_alloc<A>::builder::DEFAULT_SEED);
+  theta_intersection_alloc(const theta_intersection_alloc<A>& other);
+  theta_intersection_alloc(theta_intersection_alloc<A>&& other) noexcept;
+  ~theta_intersection_alloc();
+
+  theta_intersection_alloc<A>& operator=(theta_intersection_alloc<A> other);
+  theta_intersection_alloc<A>& operator=(theta_intersection_alloc<A>&& other);
+
+  void update(const theta_sketch_alloc<A>& sketch);
+  compact_theta_sketch_alloc<A> get_result(bool ordered = true) const;
+  bool has_result() const;
+
+private:
+  typedef typename std::allocator_traits<A>::template rebind_alloc<uint64_t> AllocU64;
+  bool is_valid_;
+  bool is_empty_;
+  uint64_t theta_;
+  uint8_t lg_size_;
+  uint64_t* keys_;
+  uint32_t num_keys_;
+  uint16_t seed_hash_;
+};
+
+// alias with default allocator for convenience
+typedef theta_intersection_alloc<std::allocator<void>> theta_intersection;
+
+} /* namespace datasketches */
+
+#include "theta_intersection_impl.hpp"
+
+# endif