From b2e499964cbd635ce440a45419d7b700d801f251 Mon Sep 17 00:00:00 2001 From: Zhiyuan Zheng Date: Mon, 7 Jan 2019 15:03:43 +0800 Subject: [PATCH 1/6] Remove heapsize dependency --- Cargo.lock | 32 -------------------------- Cargo.toml | 2 -- src/bin/repl/main.rs | 1 - src/disk_store/rocksdb.rs | 1 - src/engine/data_types/byte_slices.rs | 2 +- src/engine/data_types/types.rs | 4 ++-- src/engine/data_types/val_rows.rs | 2 +- src/engine/data_types/vec_data.rs | 7 +++--- src/ingest/buffer.rs | 7 +++++- src/ingest/raw_val.rs | 2 +- src/lib.rs | 3 --- src/mem_store/codec.rs | 4 ++-- src/mem_store/column.rs | 23 +++++++++---------- src/mem_store/partition.rs | 34 +++++++++++++++++----------- src/mem_store/raw_col.rs | 4 ++-- src/mem_store/table.rs | 29 +++++++++++------------- src/mem_store/value.rs | 11 --------- src/scheduler/disk_read_scheduler.rs | 3 --- src/scheduler/inner_locustdb.rs | 1 - 19 files changed, 63 insertions(+), 109 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cecd7dd9..3045bb9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -375,24 +375,6 @@ name = "glob" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "heapsize" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "heapsize_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", - "synstructure 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "hex" version = "0.3.2" @@ -483,8 +465,6 @@ dependencies = [ "futures-core 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "futures-executor 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "futures-util 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -980,15 +960,6 @@ dependencies = [ "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "synstructure" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "synstructure" version = "0.6.1" @@ -1182,8 +1153,6 @@ dependencies = [ "checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb" "checksum getopts 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "b900c08c1939860ce8b54dc6a89e26e00c04c380fd0e09796799bd7f12861e05" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" -"checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" -"checksum heapsize_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "46f96d52fb1564059fc97b85ef6165728cc30198ab60073bf114c66c4c89bb5d" "checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" "checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" @@ -1250,7 +1219,6 @@ dependencies = [ "checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum syn 0.15.18 (registry+https://github.com/rust-lang/crates.io-index)" = "90c39a061e2f412a9f869540471ab679e85e50c6b05604daf28bc3060f75c430" "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" -"checksum synstructure 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cf318c34a2f8381a4f3d4db2c91b45bca2b1cd8cbe56caced900647be164800c" "checksum synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a761d12e6d8dcb4dcf952a7a89b475e3a9d69e4a69307e01a470977642914bd" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" "checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" diff --git a/Cargo.toml b/Cargo.toml index baa0e303..c9a00db8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,8 +25,6 @@ futures-channel = "0.2.1" futures-core = "0.2.1" futures-executor = "0.2.1" futures-util = "0.2.1" -heapsize = "0.4.2" -heapsize_derive = "0.1.4" hex = "0.3.2" itertools = "0.5.9" lazy_static = "1.2" diff --git a/src/bin/repl/main.rs b/src/bin/repl/main.rs index e55c6269..f77780ea 100644 --- a/src/bin/repl/main.rs +++ b/src/bin/repl/main.rs @@ -3,7 +3,6 @@ extern crate clap; extern crate env_logger; extern crate failure; extern crate futures_executor; -extern crate heapsize; extern crate locustdb; extern crate log; extern crate nom; diff --git a/src/disk_store/rocksdb.rs b/src/disk_store/rocksdb.rs index 199dd683..4782b301 100644 --- a/src/disk_store/rocksdb.rs +++ b/src/disk_store/rocksdb.rs @@ -6,7 +6,6 @@ use std::str; use byteorder::{ByteOrder, BigEndian}; use capnp::{serialize, Word, message}; -use heapsize::HeapSizeOf; use self::rocksdb::*; use storage_format_capnp::*; diff --git a/src/engine/data_types/byte_slices.rs b/src/engine/data_types/byte_slices.rs index f974d209..9a424e3b 100644 --- a/src/engine/data_types/byte_slices.rs +++ b/src/engine/data_types/byte_slices.rs @@ -9,7 +9,7 @@ use engine::data_types::*; use ingest::raw_val::RawVal; -#[derive(HeapSizeOf, Debug, Clone)] +#[derive(Debug, Clone)] pub struct ByteSlices<'a> { pub row_len: usize, pub data: Vec<&'a [u8]>, diff --git a/src/engine/data_types/types.rs b/src/engine/data_types/types.rs index 6339600a..7ef49a14 100644 --- a/src/engine/data_types/types.rs +++ b/src/engine/data_types/types.rs @@ -1,6 +1,6 @@ use mem_store::*; -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, HeapSizeOf)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] pub enum EncodingType { Str, OptStr, @@ -110,7 +110,7 @@ impl EncodingType { } } -#[derive(Copy, Clone, Debug, PartialEq, HeapSizeOf)] +#[derive(Copy, Clone, Debug, PartialEq)] pub enum BasicType { String, Integer, diff --git a/src/engine/data_types/val_rows.rs b/src/engine/data_types/val_rows.rs index 13b6a187..29dbff7f 100644 --- a/src/engine/data_types/val_rows.rs +++ b/src/engine/data_types/val_rows.rs @@ -9,7 +9,7 @@ use ingest::raw_val::RawVal; use mem_store::value::Val; -#[derive(HeapSizeOf, Debug, Clone)] +#[derive(Debug, Clone)] pub struct ValRows<'a> { pub row_len: usize, pub data: Vec>, diff --git a/src/engine/data_types/vec_data.rs b/src/engine/data_types/vec_data.rs index 3312f19e..a5d5d118 100644 --- a/src/engine/data_types/vec_data.rs +++ b/src/engine/data_types/vec_data.rs @@ -8,7 +8,6 @@ use std::io::Cursor; use num::PrimInt; use byteorder::{NativeEndian, ReadBytesExt}; -use heapsize::HeapSizeOf; use ingest::raw_val::RawVal; use itertools::Itertools; @@ -16,7 +15,7 @@ use mem_store::value::Val; use engine::data_types::*; -pub trait VecData: PartialEq + Ord + Copy + Debug + Sync + Send + HeapSizeOf { +pub trait VecData: PartialEq + Ord + Copy + Debug + Sync + Send { fn unwrap<'a, 'b>(vec: &'b Data<'a>) -> &'b [T] where T: 'a; fn unwrap_mut<'a, 'b>(vec: &'b mut Data<'a>) -> &'b mut Vec where T: 'a; fn wrap_one(_value: T) -> RawVal { panic!("Can't wrap scalar of type {:?}", Self::t()) } @@ -186,7 +185,7 @@ impl CastUsize for u64 { fn cast_usize(&self) -> usize { *self as usize } } -#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Copy, Clone, HeapSizeOf)] +#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Copy, Clone)] pub enum MergeOp { TakeLeft, TakeRight, @@ -206,7 +205,7 @@ impl VecData for MergeOp { } -#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Copy, Clone, HeapSizeOf)] +#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Copy, Clone)] pub struct Premerge { pub left: u32, pub right: u32, diff --git a/src/ingest/buffer.rs b/src/ingest/buffer.rs index 4b1375b2..94a6ed89 100644 --- a/src/ingest/buffer.rs +++ b/src/ingest/buffer.rs @@ -3,9 +3,10 @@ use mem_store::raw_col::MixedCol; use ingest::raw_val::RawVal; use ingest::input_column::InputColumn; use std::cmp; +use std::mem; -#[derive(PartialEq, Debug, HeapSizeOf)] +#[derive(PartialEq, Debug)] pub struct Buffer { pub buffer: HashMap, pub length: usize, @@ -77,5 +78,9 @@ impl Buffer { pub fn len(&self) -> usize { self.length } + + pub fn heap_size_of_children(&self) -> usize { + mem::size_of::() + } } diff --git a/src/ingest/raw_val.rs b/src/ingest/raw_val.rs index 790313a1..0e99f4bf 100644 --- a/src/ingest/raw_val.rs +++ b/src/ingest/raw_val.rs @@ -2,7 +2,7 @@ use std::fmt; use engine::data_types::BasicType; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash, HeapSizeOf)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] pub enum RawVal { Int(i64), Str(String), diff --git a/src/lib.rs b/src/lib.rs index ccf311da..a515d754 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,9 +13,6 @@ extern crate futures_channel; extern crate futures_core; extern crate futures_executor; extern crate futures_util; -extern crate heapsize; -#[macro_use] -extern crate heapsize_derive; extern crate hex; extern crate itertools; #[macro_use] diff --git a/src/mem_store/codec.rs b/src/mem_store/codec.rs index 81a4c370..67397a88 100644 --- a/src/mem_store/codec.rs +++ b/src/mem_store/codec.rs @@ -2,7 +2,7 @@ use engine::*; use engine::data_types::*; use engine::planning::QueryPlanner; -#[derive(Debug, Clone, HeapSizeOf)] +#[derive(Debug, Clone)] pub struct Codec { ops: Vec, column_name: String, @@ -265,7 +265,7 @@ impl Codec { } } -#[derive(Debug, Clone, Copy, PartialEq, HeapSizeOf)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum CodecOp { Nullable, Add(EncodingType, i64), diff --git a/src/mem_store/column.rs b/src/mem_store/column.rs index 3206fae4..eb9591e2 100644 --- a/src/mem_store/column.rs +++ b/src/mem_store/column.rs @@ -6,10 +6,6 @@ use mem_store::*; use engine::data_types::*; use mem_store::lz4; -use heapsize::HeapSizeOf; - - -#[derive(HeapSizeOf)] pub struct Column { name: String, len: usize, @@ -107,6 +103,11 @@ impl Column { pub fn basic_type(&self) -> BasicType { self.codec.decoded_type() } pub fn section_encoding_type(&self, section: usize) -> EncodingType { self.data[section].encoding_type() } + + pub fn heap_size_of_children(&self) -> usize { + self.data.iter().map(|section| section.heap_size_of_children()).sum() + } + pub fn mem_tree(&self, tree: &mut MemTreeColumn, depth: usize) { if depth == 0 { return; } let size_bytes = self.heap_size_of_children(); @@ -292,16 +293,14 @@ impl DataSection { } } } -} -impl HeapSizeOf for DataSection { - fn heap_size_of_children(&self) -> usize { + pub fn heap_size_of_children(&self) -> usize { match self { - DataSection::U8(ref x) => x.heap_size_of_children(), - DataSection::U16(ref x) => x.heap_size_of_children(), - DataSection::U32(ref x) => x.heap_size_of_children(), - DataSection::U64(ref x) => x.heap_size_of_children(), - DataSection::I64(ref x) => x.heap_size_of_children(), + DataSection::U8(ref x) => x.capacity() * mem::size_of::(), + DataSection::U16(ref x) => x.capacity() * mem::size_of::(), + DataSection::U32(ref x) => x.capacity() * mem::size_of::(), + DataSection::U64(ref x) => x.capacity() * mem::size_of::(), + DataSection::I64(ref x) => x.capacity() * mem::size_of::(), DataSection::Null(_) => 0, } } diff --git a/src/mem_store/partition.rs b/src/mem_store/partition.rs index ae5394b2..de85fcfa 100644 --- a/src/mem_store/partition.rs +++ b/src/mem_store/partition.rs @@ -4,7 +4,6 @@ use std::sync::{Arc, Mutex, MutexGuard}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use disk_store::interface::*; -use heapsize::HeapSizeOf; use ingest::buffer::Buffer; use mem_store::*; use scheduler::disk_read_scheduler::DiskReadScheduler; @@ -165,19 +164,27 @@ impl Partition { self.cols.iter() .map(|handle| { let c = handle.col.lock().unwrap(); - (handle.name().to_string(), c.heap_size_of_children()) + (handle.name().to_string(), match *c { + Some(ref x) => x.heap_size_of_children(), + None => 0 + }) }) .collect() } -} -impl HeapSizeOf for Partition { - fn heap_size_of_children(&self) -> usize { - self.cols.iter().map(|handle| handle.col.lock().unwrap().heap_size_of_children()).sum() + pub fn heap_size_of_children(&self) -> usize { + self.cols.iter() + .map(|handle| { + let c = handle.col.lock().unwrap(); + match *c { + Some(ref x) => x.heap_size_of_children(), + None => 0 + } + }) + .sum() } } - pub struct ColumnHandle { key: (PartitionID, String), size_bytes: AtomicUsize, @@ -242,11 +249,12 @@ impl ColumnHandle { pub fn update_size_bytes(&self, size_bytes: usize) { self.size_bytes.store(size_bytes, Ordering::SeqCst) } -} -impl HeapSizeOf for ColumnHandle { - fn heap_size_of_children(&self) -> usize { - if self.is_resident() { self.size_bytes() } else { 0 } + pub fn heap_size_of_children(&self) -> usize { + if self.is_resident() { + self.size_bytes() + } else { + 0 + } } -} - +} \ No newline at end of file diff --git a/src/mem_store/raw_col.rs b/src/mem_store/raw_col.rs index 8108486f..ce9bb24f 100644 --- a/src/mem_store/raw_col.rs +++ b/src/mem_store/raw_col.rs @@ -7,7 +7,7 @@ use mem_store::*; use mem_store::column_builder::*; // Can eliminate this? Used by in-memory buffer. -#[derive(PartialEq, Debug, HeapSizeOf)] +#[derive(PartialEq, Debug)] pub struct MixedCol { types: ColType, data: Vec, @@ -80,7 +80,7 @@ impl Default for MixedCol { } } -#[derive(PartialEq, Debug, Copy, Clone, HeapSizeOf)] +#[derive(PartialEq, Debug, Copy, Clone)] struct ColType { contains_string: bool, contains_int: bool, diff --git a/src/mem_store/table.rs b/src/mem_store/table.rs index 0a513728..8f23c27c 100644 --- a/src/mem_store/table.rs +++ b/src/mem_store/table.rs @@ -5,7 +5,6 @@ use std::sync::Arc; use std::sync::{Mutex, RwLock}; use disk_store::interface::*; -use heapsize::HeapSizeOf; use ingest::buffer::Buffer; use ingest::input_column::InputColumn; use ingest::raw_val::RawVal; @@ -140,13 +139,25 @@ impl Table { name: self.name().to_string(), rows: partitions.iter().map(|p| p.len()).sum(), batches: partitions.len(), - batches_bytes: partitions.heap_size_of_children(), + batches_bytes: partitions.iter().map(|partition| partition.heap_size_of_children()).sum(), buffer_length: buffer.len(), buffer_bytes: buffer.heap_size_of_children(), size_per_column, } } + pub fn heap_size_of_children(&self) -> usize { + let batches_size: usize = { + let batches = self.partitions.read().unwrap(); + batches.iter().map(|(_, partition)| partition.heap_size_of_children()).sum() + }; + let buffer_size = { + let buffer = self.buffer.lock().unwrap(); + buffer.heap_size_of_children() + }; + batches_size + buffer_size + } + pub fn max_partition_id(&self) -> u64 { let partitions = self.partitions.read().unwrap(); partitions.keys().max().cloned().unwrap_or(0) @@ -167,20 +178,6 @@ fn batch_size_override(batch_size: usize, tablename: &str) -> usize { if tablename == "_meta_tables" { 1 } else if tablename == "_meta_queries" { 10 } else { batch_size } } -impl HeapSizeOf for Table { - fn heap_size_of_children(&self) -> usize { - let batches_size = { - let batches = self.partitions.read().unwrap(); - batches.heap_size_of_children() - }; - let buffer_size = { - let buffer = self.buffer.lock().unwrap(); - buffer.heap_size_of_children() - }; - batches_size + buffer_size - } -} - #[derive(PartialEq, Debug, Clone)] pub struct Metadata { pub name: String, diff --git a/src/mem_store/value.rs b/src/mem_store/value.rs index e6b80f1f..3146666d 100644 --- a/src/mem_store/value.rs +++ b/src/mem_store/value.rs @@ -1,5 +1,4 @@ use std::fmt; -use heapsize::HeapSizeOf; use std::convert::From; use ingest::raw_val::RawVal; @@ -23,16 +22,6 @@ impl<'a> fmt::Display for Val<'a> { } } -impl<'a> HeapSizeOf for Val<'a> { - fn heap_size_of_children(&self) -> usize { - use self::Val::*; - match *self { - Null | Bool(_) | Integer(_) => 0, - Str(r) => r.heap_size_of_children(), - } - } -} - impl<'a> From<()> for Val<'a> { fn from(_: ()) -> Val<'a> { Val::Null diff --git a/src/scheduler/disk_read_scheduler.rs b/src/scheduler/disk_read_scheduler.rs index d0784130..20f05495 100644 --- a/src/scheduler/disk_read_scheduler.rs +++ b/src/scheduler/disk_read_scheduler.rs @@ -6,9 +6,6 @@ use std::sync::{Arc, Mutex, Condvar}; use std::collections::VecDeque; use std_semaphore::Semaphore; -#[allow(unused_imports)] -use heapsize::HeapSizeOf; - use disk_store::interface::DiskStore; use disk_store::interface::PartitionID; use mem_store::*; diff --git a/src/scheduler/inner_locustdb.rs b/src/scheduler/inner_locustdb.rs index 120b88ad..26730df4 100644 --- a/src/scheduler/inner_locustdb.rs +++ b/src/scheduler/inner_locustdb.rs @@ -8,7 +8,6 @@ use std::time::Duration; use futures_core::*; use futures_channel::oneshot; -use heapsize::HeapSizeOf; use time; use disk_store::interface::*; From d25379fff785f71a346cd450a6b2e91aa263507c Mon Sep 17 00:00:00 2001 From: Zhiyuan Zheng Date: Mon, 7 Jan 2019 15:03:58 +0800 Subject: [PATCH 2/6] Update toolchain version --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index 7cd3a803..fd1ca4c0 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2018-12-30 +nightly-2018-12-31 From dae76943986909412a454dea62f251cb8ad3adc6 Mon Sep 17 00:00:00 2001 From: Zhiyuan Zheng Date: Tue, 8 Jan 2019 10:49:12 +0800 Subject: [PATCH 3/6] Fix the heap_size_of_children of Buffer object. --- .gitignore | 3 +++ src/ingest/buffer.rs | 7 ++++--- src/mem_store/raw_col.rs | 5 +++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index f42ea6dc..48d20f54 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ rusty-tags.vi # default rocksdb data dir rocksdb + +# OS X +.DS_Store diff --git a/src/ingest/buffer.rs b/src/ingest/buffer.rs index 94a6ed89..52ae4d09 100644 --- a/src/ingest/buffer.rs +++ b/src/ingest/buffer.rs @@ -3,8 +3,6 @@ use mem_store::raw_col::MixedCol; use ingest::raw_val::RawVal; use ingest::input_column::InputColumn; use std::cmp; -use std::mem; - #[derive(PartialEq, Debug)] pub struct Buffer { @@ -80,7 +78,10 @@ impl Buffer { } pub fn heap_size_of_children(&self) -> usize { - mem::size_of::() + self.buffer.iter().map(|(_, v)| { + // Currently does not take into account the memory of String. + v.heap_size_of_children() + }).sum() } } diff --git a/src/mem_store/raw_col.rs b/src/mem_store/raw_col.rs index ce9bb24f..99a6c3e9 100644 --- a/src/mem_store/raw_col.rs +++ b/src/mem_store/raw_col.rs @@ -1,6 +1,7 @@ use std::iter::repeat; use std::ops::BitOr; use std::sync::Arc; +use std::mem; use ingest::raw_val::RawVal; use mem_store::*; @@ -69,6 +70,10 @@ impl MixedCol { Arc::new(Column::null(name, self.data.len())) } } + + pub fn heap_size_of_children(&self) -> usize { + mem::size_of::() + self.data.capacity() * mem::size_of::() + } } impl Default for MixedCol { From 70597e65174403878f70e3800831885230a16f12 Mon Sep 17 00:00:00 2001 From: Zhiyuan Zheng Date: Tue, 8 Jan 2019 12:34:23 +0800 Subject: [PATCH 4/6] Use more accurate size of RawVal --- src/ingest/raw_val.rs | 10 ++++++++++ src/mem_store/raw_col.rs | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/ingest/raw_val.rs b/src/ingest/raw_val.rs index 0e99f4bf..8c6c90d0 100644 --- a/src/ingest/raw_val.rs +++ b/src/ingest/raw_val.rs @@ -1,4 +1,5 @@ use std::fmt; +use std::mem; use engine::data_types::BasicType; @@ -17,6 +18,15 @@ impl RawVal { RawVal::Null => BasicType::Null, } } + + pub fn heap_size_of_children(&self) -> usize { + match *self { + RawVal::Int(_) => mem::size_of::(), + RawVal::Str(ref s) => s.capacity() * mem::size_of::(), + RawVal::Null => mem::size_of::() + } + } + } impl fmt::Display for RawVal { diff --git a/src/mem_store/raw_col.rs b/src/mem_store/raw_col.rs index 99a6c3e9..d3a782be 100644 --- a/src/mem_store/raw_col.rs +++ b/src/mem_store/raw_col.rs @@ -72,7 +72,10 @@ impl MixedCol { } pub fn heap_size_of_children(&self) -> usize { - mem::size_of::() + self.data.capacity() * mem::size_of::() + let data_size = self.data.iter().map(|v| v.heap_size_of_children()).sum::(); + let type_size = mem::size_of::(); + + data_size + type_size } } From e9cf828bc3782a13155ddd1c2594a56b69a91764 Mon Sep 17 00:00:00 2001 From: Zhiyuan Zheng Date: Tue, 8 Jan 2019 12:57:55 +0800 Subject: [PATCH 5/6] Take RawVal itself into account. --- src/ingest/raw_val.rs | 2 +- src/mem_store/raw_col.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ingest/raw_val.rs b/src/ingest/raw_val.rs index 8c6c90d0..be84af5d 100644 --- a/src/ingest/raw_val.rs +++ b/src/ingest/raw_val.rs @@ -23,7 +23,7 @@ impl RawVal { match *self { RawVal::Int(_) => mem::size_of::(), RawVal::Str(ref s) => s.capacity() * mem::size_of::(), - RawVal::Null => mem::size_of::() + RawVal::Null => 0, } } diff --git a/src/mem_store/raw_col.rs b/src/mem_store/raw_col.rs index d3a782be..71febfad 100644 --- a/src/mem_store/raw_col.rs +++ b/src/mem_store/raw_col.rs @@ -72,7 +72,8 @@ impl MixedCol { } pub fn heap_size_of_children(&self) -> usize { - let data_size = self.data.iter().map(|v| v.heap_size_of_children()).sum::(); + let data_size = self.data.iter().map(|v| v.heap_size_of_children()).sum::() + + self.data.capacity() * mem::size_of::(); let type_size = mem::size_of::(); data_size + type_size From e4f40e4de8c004fbbd92f1c7f004347780338d39 Mon Sep 17 00:00:00 2001 From: Zhiyuan Zheng Date: Tue, 8 Jan 2019 13:25:11 +0800 Subject: [PATCH 6/6] size_of enum will account it. --- src/ingest/raw_val.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingest/raw_val.rs b/src/ingest/raw_val.rs index be84af5d..23aec5e2 100644 --- a/src/ingest/raw_val.rs +++ b/src/ingest/raw_val.rs @@ -21,7 +21,7 @@ impl RawVal { pub fn heap_size_of_children(&self) -> usize { match *self { - RawVal::Int(_) => mem::size_of::(), + RawVal::Int(_) => 0, RawVal::Str(ref s) => s.capacity() * mem::size_of::(), RawVal::Null => 0, }