Remove the heapsize dependency.

.gitignore

@@ -22,3 +22,6 @@ rusty-tags.vi
 
 # default rocksdb data dir
 rocksdb
+
+# OS X
+.DS_Store

Cargo.toml

@@ -25,8 +25,6 @@ futures-channel = "0.2.1"
 futures-core = "0.2.1"
 futures-executor = "0.2.1"
 futures-util = "0.2.1"
-heapsize = "0.4.2"
-heapsize_derive = "0.1.4"
 hex = "0.3.2"
 itertools = "0.5.9"
 lazy_static = "1.2"

rust-toolchain

@@ -1 +1 @@
-nightly-2018-12-30
+nightly-2018-12-31

src/bin/repl/main.rs

@@ -3,7 +3,6 @@ extern crate clap;
 extern crate env_logger;
 extern crate failure;
 extern crate futures_executor;
-extern crate heapsize;
 extern crate locustdb;
 extern crate log;
 extern crate nom;

src/disk_store/rocksdb.rs

@@ -6,7 +6,6 @@ use std::str;
 
 use byteorder::{ByteOrder, BigEndian};
 use capnp::{serialize, Word, message};
-use heapsize::HeapSizeOf;
 use self::rocksdb::*;
 use storage_format_capnp::*;
 

src/engine/data_types/byte_slices.rs

@@ -9,7 +9,7 @@ use engine::data_types::*;
 use ingest::raw_val::RawVal;
 
 
-#[derive(HeapSizeOf, Debug, Clone)]
+#[derive(Debug, Clone)]
 pub struct ByteSlices<'a> {
     pub row_len: usize,
     pub data: Vec<&'a [u8]>,

src/engine/data_types/types.rs

@@ -1,6 +1,6 @@
 use mem_store::*;
 
-#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, HeapSizeOf)]
+#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
 pub enum EncodingType {
     Str,
     OptStr,
@@ -110,7 +110,7 @@ impl EncodingType {
     }
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, HeapSizeOf)]
+#[derive(Copy, Clone, Debug, PartialEq)]
 pub enum BasicType {
     String,
     Integer,

src/engine/data_types/val_rows.rs

@@ -9,7 +9,7 @@ use ingest::raw_val::RawVal;
 use mem_store::value::Val;
 
 
-#[derive(HeapSizeOf, Debug, Clone)]
+#[derive(Debug, Clone)]
 pub struct ValRows<'a> {
     pub row_len: usize,
     pub data: Vec<Val<'a>>,

src/engine/data_types/vec_data.rs

@@ -8,15 +8,14 @@ use std::io::Cursor;
 
 use num::PrimInt;
 use byteorder::{NativeEndian, ReadBytesExt};
-use heapsize::HeapSizeOf;
 use ingest::raw_val::RawVal;
 use itertools::Itertools;
 
 use mem_store::value::Val;
 use engine::data_types::*;
 
 
-pub trait VecData<T>: PartialEq + Ord + Copy + Debug + Sync + Send + HeapSizeOf {
+pub trait VecData<T>: PartialEq + Ord + Copy + Debug + Sync + Send {
     fn unwrap<'a, 'b>(vec: &'b Data<'a>) -> &'b [T] where T: 'a;
     fn unwrap_mut<'a, 'b>(vec: &'b mut Data<'a>) -> &'b mut Vec<T> where T: 'a;
     fn wrap_one(_value: T) -> RawVal { panic!("Can't wrap scalar of type {:?}", Self::t()) }
@@ -186,7 +185,7 @@ impl CastUsize for u64 {
     fn cast_usize(&self) -> usize { *self as usize }
 }
 
-#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Copy, Clone, HeapSizeOf)]
+#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Copy, Clone)]
 pub enum MergeOp {
     TakeLeft,
     TakeRight,
@@ -206,7 +205,7 @@ impl VecData<MergeOp> for MergeOp {
 }
 
 
-#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Copy, Clone, HeapSizeOf)]
+#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Copy, Clone)]
 pub struct Premerge {
     pub left: u32,
     pub right: u32,

src/ingest/buffer.rs

@@ -4,8 +4,7 @@ use ingest::raw_val::RawVal;
 use ingest::input_column::InputColumn;
 use std::cmp;
 
-
-#[derive(PartialEq, Debug, HeapSizeOf)]
+#[derive(PartialEq, Debug)]
 pub struct Buffer {
     pub buffer: HashMap<String, MixedCol>,
     pub length: usize,
@@ -77,5 +76,12 @@ impl Buffer {
     pub fn len(&self) -> usize {
         self.length
     }
+
+    pub fn heap_size_of_children(&self) -> usize {
+        self.buffer.iter().map(|(_, v)| {
+            // Currently does not take into account the memory of String.
+            v.heap_size_of_children()
+        }).sum()
+    }
 }
 

src/ingest/raw_val.rs

@@ -1,8 +1,9 @@
 use std::fmt;
+use std::mem;
 use engine::data_types::BasicType;
 
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash, HeapSizeOf)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)]
 pub enum RawVal {
     Int(i64),
     Str(String),
@@ -17,6 +18,15 @@ impl RawVal {
             RawVal::Null => BasicType::Null,
         }
     }
+
+    pub fn heap_size_of_children(&self) -> usize {
+        match *self {
+            RawVal::Int(_) => mem::size_of::<i64>(),
+            RawVal::Str(ref s) => s.capacity() * mem::size_of::<u8>(),
+            RawVal::Null => 0,
+        }
+    }
+
 }
 
 impl fmt::Display for RawVal {

src/lib.rs

@@ -13,9 +13,6 @@ extern crate futures_channel;
 extern crate futures_core;
 extern crate futures_executor;
 extern crate futures_util;
-extern crate heapsize;
-#[macro_use]
-extern crate heapsize_derive;
 extern crate hex;
 extern crate itertools;
 #[macro_use]

src/mem_store/codec.rs

@@ -2,7 +2,7 @@ use engine::*;
 use engine::data_types::*;
 use engine::planning::QueryPlanner;
 
-#[derive(Debug, Clone, HeapSizeOf)]
+#[derive(Debug, Clone)]
 pub struct Codec {
     ops: Vec<CodecOp>,
     column_name: String,
@@ -265,7 +265,7 @@ impl Codec {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, HeapSizeOf)]
+#[derive(Debug, Clone, Copy, PartialEq)]
 pub enum CodecOp {
     Nullable,
     Add(EncodingType, i64),

src/mem_store/column.rs

@@ -6,10 +6,6 @@ use mem_store::*;
 use engine::data_types::*;
 use mem_store::lz4;
 
-use heapsize::HeapSizeOf;
-
-
-#[derive(HeapSizeOf)]
 pub struct Column {
     name: String,
     len: usize,
@@ -107,6 +103,11 @@ impl Column {
     pub fn basic_type(&self) -> BasicType { self.codec.decoded_type() }
     pub fn section_encoding_type(&self, section: usize) -> EncodingType { self.data[section].encoding_type() }
 
+
+    pub fn heap_size_of_children(&self) -> usize {
+        self.data.iter().map(|section| section.heap_size_of_children()).sum()
+    }
+
     pub fn mem_tree(&self, tree: &mut MemTreeColumn, depth: usize) {
         if depth == 0 { return; }
         let size_bytes = self.heap_size_of_children();
@@ -292,16 +293,14 @@ impl DataSection {
             }
         }
     }
-}
 
-impl HeapSizeOf for DataSection {
-    fn heap_size_of_children(&self) -> usize {
+    pub fn heap_size_of_children(&self) -> usize {
         match self {
-            DataSection::U8(ref x) => x.heap_size_of_children(),
-            DataSection::U16(ref x) => x.heap_size_of_children(),
-            DataSection::U32(ref x) => x.heap_size_of_children(),
-            DataSection::U64(ref x) => x.heap_size_of_children(),
-            DataSection::I64(ref x) => x.heap_size_of_children(),
+            DataSection::U8(ref x) => x.capacity() * mem::size_of::<u8>(),
+            DataSection::U16(ref x) => x.capacity() * mem::size_of::<u16>(),
+            DataSection::U32(ref x) => x.capacity() * mem::size_of::<u32>(),
+            DataSection::U64(ref x) => x.capacity() * mem::size_of::<u64>(),
+            DataSection::I64(ref x) => x.capacity() * mem::size_of::<i64>(),
             DataSection::Null(_) => 0,
         }
     }

src/mem_store/partition.rs

@@ -4,7 +4,6 @@ use std::sync::{Arc, Mutex, MutexGuard};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 
 use disk_store::interface::*;
-use heapsize::HeapSizeOf;
 use ingest::buffer::Buffer;
 use mem_store::*;
 use scheduler::disk_read_scheduler::DiskReadScheduler;
@@ -165,19 +164,27 @@ impl Partition {
         self.cols.iter()
             .map(|handle| {
                 let c = handle.col.lock().unwrap();
-                (handle.name().to_string(), c.heap_size_of_children())
+                (handle.name().to_string(), match *c {
+                    Some(ref x) => x.heap_size_of_children(),
+                    None => 0
+                })
             })
             .collect()
     }
-}
 
-impl HeapSizeOf for Partition {
-    fn heap_size_of_children(&self) -> usize {
-        self.cols.iter().map(|handle| handle.col.lock().unwrap().heap_size_of_children()).sum()
+    pub fn heap_size_of_children(&self) -> usize {
+        self.cols.iter()
+            .map(|handle| {
+                let c = handle.col.lock().unwrap();
+                match *c {
+                    Some(ref x) => x.heap_size_of_children(),
+                    None => 0
+                }
+            })
+            .sum()
     }
 }
 
-
 pub struct ColumnHandle {
     key: (PartitionID, String),
     size_bytes: AtomicUsize,
@@ -242,11 +249,12 @@ impl ColumnHandle {
     pub fn update_size_bytes(&self, size_bytes: usize) {
         self.size_bytes.store(size_bytes, Ordering::SeqCst)
     }
-}
 
-impl HeapSizeOf for ColumnHandle {
-    fn heap_size_of_children(&self) -> usize {
-        if self.is_resident() { self.size_bytes() } else { 0 }
+    pub fn heap_size_of_children(&self) -> usize {
+        if self.is_resident() { 
+            self.size_bytes() 
+        } else { 
+            0 
+        }
     }
-}
-
+}

src/mem_store/raw_col.rs

@@ -1,13 +1,14 @@
 use std::iter::repeat;
 use std::ops::BitOr;
 use std::sync::Arc;
+use std::mem;
 
 use ingest::raw_val::RawVal;
 use mem_store::*;
 use mem_store::column_builder::*;
 
 // Can eliminate this? Used by in-memory buffer.
-#[derive(PartialEq, Debug, HeapSizeOf)]
+#[derive(PartialEq, Debug)]
 pub struct MixedCol {
     types: ColType,
     data: Vec<RawVal>,
@@ -69,6 +70,14 @@ impl MixedCol {
             Arc::new(Column::null(name, self.data.len()))
         }
     }
+
+    pub fn heap_size_of_children(&self) -> usize {
+        let data_size = self.data.iter().map(|v| v.heap_size_of_children()).sum::<usize>()
+            + self.data.capacity() * mem::size_of::<RawVal>();
+        let type_size = mem::size_of::<ColType>();
+
+        data_size + type_size
+    }
 }
 
 impl Default for MixedCol {
@@ -80,7 +89,7 @@ impl Default for MixedCol {
     }
 }
 
-#[derive(PartialEq, Debug, Copy, Clone, HeapSizeOf)]
+#[derive(PartialEq, Debug, Copy, Clone)]
 struct ColType {
     contains_string: bool,
     contains_int: bool,