From da666283e7c83b22dc32681274645c0cf6fea93d Mon Sep 17 00:00:00 2001 From: David Frank Date: Thu, 21 Nov 2024 15:29:25 +0000 Subject: [PATCH 1/9] perf: Add new efficient APIs read_unsafe and read_to_vec I found that a source of significant performance loss is the read method of Memory. The read method takes a mutable buffer which it fills with values read from the stable memory. According to Rust rules, the buffer passed to read must be initialized before it's passed to read (buffers containing uninitialized values are unsound and can cause UB). The usual pattern is to create a properly sized Vec, eg. by using `vec![0; size]` or `vec.resize(size, 0)` and pass that to `read`. However, initializing the bytes with values that get overwritten by `read` is only necessary in order to be sound and requires significant number of instructions. This PR introduces a new method `read_unsafe` which allows passing in a raw pointer and a `count` parameter. Implementations can be more efficient by reading directly and skipping initialization. This can lead to instruction reductions of up to 40%. The PR also introduces a helper method `read_to_vec` which is a safe wrapper around `read_unsafe` for the most common use-case: reading into a `Vec`. Clients can for example pass an empty `Vec` and profit from the extra efficiency without having to call unsafe methods. --- canbench_results.yml | 212 ++++++++++++++++++++-------------------- src/base_vec.rs | 7 +- src/btreemap/node.rs | 6 +- src/btreemap/node/io.rs | 32 ++++-- src/btreemap/node/v1.rs | 5 +- src/btreemap/node/v2.rs | 3 +- src/cell.rs | 6 +- src/ic0_memory.rs | 6 ++ src/lib.rs | 50 +++++++++- src/log.rs | 6 +- src/memory_manager.rs | 14 ++- src/vec_mem.rs | 19 ++++ 12 files changed, 223 insertions(+), 143 deletions(-) diff --git a/canbench_results.yml b/canbench_results.yml index d80974e5..abc8760e 100644 --- a/canbench_results.yml +++ b/canbench_results.yml @@ -1,595 +1,595 @@ benches: btreemap_get_blob_128_1024: total: - instructions: 868028993 + instructions: 826371552 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_128_1024_v2: total: - instructions: 970754065 + instructions: 912394951 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_16_1024: total: - instructions: 304555831 + instructions: 300853830 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_16_1024_v2: total: - instructions: 409386098 + instructions: 376695579 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_256_1024: total: - instructions: 1399133930 + instructions: 1329458474 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_256_1024_v2: total: - instructions: 1499851078 + instructions: 1416464604 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_32_1024: total: - instructions: 347918329 + instructions: 329605716 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_32_1024_v2: total: - instructions: 452317746 + instructions: 410560225 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_4_1024: total: - instructions: 194406501 + instructions: 180863991 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_4_1024_v2: total: - instructions: 289345080 + instructions: 269645388 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_512_1024: total: - instructions: 2454262332 + instructions: 2338381852 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_512_1024_v2: total: - instructions: 2556213321 + instructions: 2421706283 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_64_1024: total: - instructions: 599847298 + instructions: 573701035 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_64_1024_v2: total: - instructions: 712044951 + instructions: 662936241 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_1024: total: - instructions: 231572100 + instructions: 208888626 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_1024_v2: total: - instructions: 323197606 + instructions: 294374626 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_u64: total: - instructions: 204171772 + instructions: 196845655 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_u64_v2: total: - instructions: 311528236 + instructions: 298384613 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_blob_8: total: - instructions: 186152093 + instructions: 184588240 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_blob_8_v2: total: - instructions: 273254410 + instructions: 266224360 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_u64: total: - instructions: 187408768 + instructions: 185443402 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_u64_v2: total: - instructions: 279916538 + instructions: 272432249 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_insert_10mib_values: total: - instructions: 141252389 + instructions: 82723368 heap_increase: 0 stable_memory_increase: 32 scopes: {} btreemap_insert_blob_1024_128: total: - instructions: 5083095809 + instructions: 4905425072 heap_increase: 0 stable_memory_increase: 262 scopes: {} btreemap_insert_blob_1024_128_v2: total: - instructions: 5196791890 + instructions: 4968538080 heap_increase: 0 stable_memory_increase: 196 scopes: {} btreemap_insert_blob_1024_16: total: - instructions: 5039894252 + instructions: 4893737868 heap_increase: 0 stable_memory_increase: 241 scopes: {} btreemap_insert_blob_1024_16_v2: total: - instructions: 5154652816 + instructions: 4947593670 heap_increase: 0 stable_memory_increase: 181 scopes: {} btreemap_insert_blob_1024_256: total: - instructions: 5109831287 + instructions: 4920366269 heap_increase: 0 stable_memory_increase: 292 scopes: {} btreemap_insert_blob_1024_256_v2: total: - instructions: 5222715261 + instructions: 4995926965 heap_increase: 0 stable_memory_increase: 219 scopes: {} btreemap_insert_blob_1024_32: total: - instructions: 5048498828 + instructions: 4905322013 heap_increase: 0 stable_memory_increase: 239 scopes: {} btreemap_insert_blob_1024_32_v2: total: - instructions: 5165315603 + instructions: 4965639927 heap_increase: 0 stable_memory_increase: 180 scopes: {} btreemap_insert_blob_1024_4: total: - instructions: 4938758192 + instructions: 4794955186 heap_increase: 0 stable_memory_increase: 235 scopes: {} btreemap_insert_blob_1024_4_v2: total: - instructions: 5053526974 + instructions: 4844178539 heap_increase: 0 stable_memory_increase: 176 scopes: {} btreemap_insert_blob_1024_512: total: - instructions: 5232116884 + instructions: 5009944961 heap_increase: 0 stable_memory_increase: 348 scopes: {} btreemap_insert_blob_1024_512_v2: total: - instructions: 5346066549 + instructions: 5084945715 heap_increase: 0 stable_memory_increase: 261 scopes: {} btreemap_insert_blob_1024_64: total: - instructions: 5080908184 + instructions: 4912581068 heap_increase: 0 stable_memory_increase: 250 scopes: {} btreemap_insert_blob_1024_64_v2: total: - instructions: 5196471697 + instructions: 4985803250 heap_increase: 0 stable_memory_increase: 188 scopes: {} btreemap_insert_blob_1024_8: total: - instructions: 5022590452 + instructions: 4856246096 heap_increase: 0 stable_memory_increase: 237 scopes: {} btreemap_insert_blob_1024_8_v2: total: - instructions: 5136760933 + instructions: 4933641074 heap_increase: 0 stable_memory_increase: 178 scopes: {} btreemap_insert_blob_128_1024: total: - instructions: 1437432650 + instructions: 1273170444 heap_increase: 0 stable_memory_increase: 260 scopes: {} btreemap_insert_blob_128_1024_v2: total: - instructions: 1543573613 + instructions: 1371510921 heap_increase: 0 stable_memory_increase: 195 scopes: {} btreemap_insert_blob_16_1024: total: - instructions: 819015110 + instructions: 692954040 heap_increase: 0 stable_memory_increase: 215 scopes: {} btreemap_insert_blob_16_1024_v2: total: - instructions: 929386468 + instructions: 788011123 heap_increase: 0 stable_memory_increase: 161 scopes: {} btreemap_insert_blob_256_1024: total: - instructions: 2007916344 + instructions: 1827822057 heap_increase: 0 stable_memory_increase: 292 scopes: {} btreemap_insert_blob_256_1024_v2: total: - instructions: 2117490329 + instructions: 1922268722 heap_increase: 0 stable_memory_increase: 219 scopes: {} btreemap_insert_blob_32_1024: total: - instructions: 870306996 + instructions: 733238813 heap_increase: 0 stable_memory_increase: 230 scopes: {} btreemap_insert_blob_32_1024_v2: total: - instructions: 980735900 + instructions: 828693177 heap_increase: 0 stable_memory_increase: 173 scopes: {} btreemap_insert_blob_4_1024: total: - instructions: 615105126 + instructions: 494283887 heap_increase: 0 stable_memory_increase: 123 scopes: {} btreemap_insert_blob_4_1024_v2: total: - instructions: 714364544 + instructions: 592473788 heap_increase: 0 stable_memory_increase: 92 scopes: {} btreemap_insert_blob_512_1024: total: - instructions: 3140701710 + instructions: 2918030428 heap_increase: 0 stable_memory_increase: 351 scopes: {} btreemap_insert_blob_512_1024_v2: total: - instructions: 3248440359 + instructions: 3003589075 heap_increase: 0 stable_memory_increase: 263 scopes: {} btreemap_insert_blob_64_1024: total: - instructions: 1136783783 + instructions: 999684307 heap_increase: 0 stable_memory_increase: 245 scopes: {} btreemap_insert_blob_64_1024_v2: total: - instructions: 1254935822 + instructions: 1095310543 heap_increase: 0 stable_memory_increase: 183 scopes: {} btreemap_insert_blob_8_1024: total: - instructions: 737978721 + instructions: 603879508 heap_increase: 0 stable_memory_increase: 183 scopes: {} btreemap_insert_blob_8_1024_v2: total: - instructions: 840828948 + instructions: 706921520 heap_increase: 0 stable_memory_increase: 138 scopes: {} btreemap_insert_blob_8_u64: total: - instructions: 333681568 + instructions: 328972005 heap_increase: 0 stable_memory_increase: 6 scopes: {} btreemap_insert_blob_8_u64_v2: total: - instructions: 448187595 + instructions: 440950328 heap_increase: 0 stable_memory_increase: 4 scopes: {} btreemap_insert_u64_blob_8: total: - instructions: 343283004 + instructions: 347528237 heap_increase: 0 stable_memory_increase: 7 scopes: {} btreemap_insert_u64_blob_8_v2: total: - instructions: 429045170 + instructions: 431734631 heap_increase: 0 stable_memory_increase: 5 scopes: {} btreemap_insert_u64_u64: total: - instructions: 353426417 + instructions: 353813009 heap_increase: 0 stable_memory_increase: 7 scopes: {} btreemap_insert_u64_u64_v2: total: - instructions: 441738753 + instructions: 440489004 heap_increase: 0 stable_memory_increase: 6 scopes: {} btreemap_iter_10mib_values: total: - instructions: 17054815 + instructions: 11431732 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_count_10mib_values: total: - instructions: 492818 + instructions: 490726 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_count_small_values: total: - instructions: 9854951 + instructions: 9637293 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_rev_10mib_values: total: - instructions: 17052720 + instructions: 11430523 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_rev_small_values: total: - instructions: 13980402 + instructions: 14754706 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_small_values: total: - instructions: 13983601 + instructions: 14713901 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_10mib_values: total: - instructions: 482392 + instructions: 477180 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_rev_10mib_values: total: - instructions: 484745 + instructions: 479533 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_rev_small_values: total: - instructions: 10226743 + instructions: 10009085 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_small_values: total: - instructions: 9990906 + instructions: 9773248 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_read_every_third_value_from_range: total: - instructions: 112238714 + instructions: 85060540 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_read_keys_from_range: total: - instructions: 112238714 + instructions: 85060540 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_128_1024: total: - instructions: 1783414773 + instructions: 1536789523 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_128_1024_v2: total: - instructions: 1942678397 + instructions: 1678979750 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_16_1024: total: - instructions: 1017479433 + instructions: 810568836 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_16_1024_v2: total: - instructions: 1170176089 + instructions: 947573091 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_256_1024: total: - instructions: 2454002394 + instructions: 2182801039 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_256_1024_v2: total: - instructions: 2606407435 + instructions: 2316460898 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_32_1024: total: - instructions: 1095183445 + instructions: 876948171 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_32_1024_v2: total: - instructions: 1252648744 + instructions: 1015025410 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_4_1024: total: - instructions: 617902738 + instructions: 484434371 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_4_1024_v2: total: - instructions: 735874430 + instructions: 600274873 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_512_1024: total: - instructions: 3849259424 + instructions: 3529926666 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_512_1024_v2: total: - instructions: 4003887427 + instructions: 3653472751 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_64_1024: total: - instructions: 1427599410 + instructions: 1204953533 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_64_1024_v2: total: - instructions: 1592814332 + instructions: 1345064138 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_1024: total: - instructions: 816476142 + instructions: 633622845 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_1024_v2: total: - instructions: 952832423 + instructions: 767528036 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_u64: total: - instructions: 438199023 + instructions: 434105759 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_u64_v2: total: - instructions: 591309106 + instructions: 585250331 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_blob_8: total: - instructions: 485820315 + instructions: 494615895 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_blob_8_v2: total: - instructions: 606724491 + instructions: 615668380 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_u64: total: - instructions: 506491112 + instructions: 508864341 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_u64_v2: total: - instructions: 637006929 + instructions: 639517433 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_10mib_values: total: - instructions: 17078596 + instructions: 11430320 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_rev_10mib_values: total: - instructions: 17077387 + instructions: 11429111 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_rev_small_values: total: - instructions: 15212366 + instructions: 14684044 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_small_values: total: - instructions: 15171561 + instructions: 14643239 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -613,43 +613,43 @@ benches: scopes: {} vec_get_blob_128: total: - instructions: 21384965 + instructions: 18554323 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_blob_16: total: - instructions: 9821962 + instructions: 8521818 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_blob_32: total: - instructions: 10694461 + instructions: 9170075 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_blob_4: total: - instructions: 5814638 + instructions: 5363359 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_blob_64: total: - instructions: 15512039 + instructions: 13553328 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_blob_8: total: - instructions: 6947467 + instructions: 6146813 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_u64: total: - instructions: 6220307 + instructions: 5540316 heap_increase: 0 stable_memory_increase: 0 scopes: {} diff --git a/src/base_vec.rs b/src/base_vec.rs index 8a3a3b45..814de59e 100644 --- a/src/base_vec.rs +++ b/src/base_vec.rs @@ -34,7 +34,7 @@ use crate::storable::{bounds, bytes_to_store_size_bounded}; use crate::{ read_u32, read_u64, safe_write, write, write_u32, write_u64, Address, GrowFailed, Memory, - Storable, + MemoryExt, Storable, }; use std::borrow::{Borrow, Cow}; use std::cmp::min; @@ -245,11 +245,10 @@ impl BaseVec { } /// Reads the item at the specified index without any bound checks. - fn read_entry_to(&self, index: u64, buf: &mut std::vec::Vec) { + fn read_entry_to(&self, index: u64, buf: &mut Vec) { let offset = DATA_OFFSET + slot_size::() as u64 * index; let (data_offset, data_size) = self.read_entry_size(offset); - buf.resize(data_size, 0); - self.memory.read(data_offset, &mut buf[..]); + self.memory.read_to_vec(data_offset, buf, data_size); } /// Sets the vector's length. diff --git a/src/btreemap/node.rs b/src/btreemap/node.rs index ff70ccef..ef05037e 100644 --- a/src/btreemap/node.rs +++ b/src/btreemap/node.rs @@ -3,7 +3,7 @@ use crate::{ read_struct, read_u32, read_u64, storable::Storable, types::{Address, Bytes}, - write, write_struct, write_u32, Memory, + write, write_struct, write_u32, Memory, MemoryExt, }; use std::borrow::{Borrow, Cow}; use std::cell::OnceCell; @@ -200,8 +200,8 @@ impl Node { }; let value_len = read_u32(&reader, Address::from(offset.get())) as usize; - let mut bytes = vec![0; value_len]; - reader.read((offset + U32_SIZE).get(), &mut bytes); + let mut bytes = vec![]; + reader.read_to_vec((offset + U32_SIZE).get(), &mut bytes, value_len); bytes } diff --git a/src/btreemap/node/io.rs b/src/btreemap/node/io.rs index 45ebec27..c69a6475 100644 --- a/src/btreemap/node/io.rs +++ b/src/btreemap/node/io.rs @@ -18,20 +18,21 @@ pub struct NodeReader<'a, M: Memory> { // Note: The `Memory` interface is implemented so that helper methods such `read_u32`, // `read_struct`, etc. can be used with a `NodeReader` directly. impl<'a, M: Memory> Memory for NodeReader<'a, M> { - fn read(&self, offset: u64, dst: &mut [u8]) { + unsafe fn read_unsafe(&self, offset: u64, dst: *mut u8, count: usize) { // If the read is only in the initial page, then read it directly in one go. // This is a performance enhancement to avoid the cost of creating a `NodeIterator`. - if (offset + dst.len() as u64) < self.page_size.get() as u64 { - self.memory.read(self.address.get() + offset, dst); + if (offset + count as u64) < self.page_size.get() as u64 { + self.memory + .read_unsafe(self.address.get() + offset, dst, count); return; } - // The read is split across several pages. Create a `NodeIterator` to to read from + // The read is split across several pages. Create a `NodeIterator` to read from // each of the individual pages. let iter = NodeIterator::new( VirtualSegment { address: Address::from(offset), - length: Bytes::from(dst.len() as u64), + length: Bytes::from(count as u64), }, Bytes::from(self.page_size.get()), ); @@ -43,22 +44,33 @@ impl<'a, M: Memory> Memory for NodeReader<'a, M> { length, } in iter { + // SAFETY: read_unsafe() is safe to call iff bytes_read + length <= count since the + // caller guarantees that we can write `count` number of bytes from `dst`. + assert!(bytes_read + length.get() as usize <= count); if page_idx == 0 { - self.memory.read( + self.memory.read_unsafe( (self.address + offset).get(), - &mut dst[bytes_read as usize..(bytes_read + length.get()) as usize], + dst.add(bytes_read), + length.get() as usize, ); } else { - self.memory.read( + self.memory.read_unsafe( (self.overflows[page_idx - 1] + offset).get(), - &mut dst[bytes_read as usize..(bytes_read + length.get()) as usize], + dst.add(bytes_read), + length.get() as usize, ); } - bytes_read += length.get(); + bytes_read += length.get() as usize; } } + #[inline] + fn read(&self, offset: u64, dst: &mut [u8]) { + // SAFETY: since dst is dst.len() long, it fulfills the safety requirements of read_unsafe. + unsafe { self.read_unsafe(offset, dst.as_mut_ptr(), dst.len()) } + } + fn write(&self, _: u64, _: &[u8]) { unreachable!("NodeReader does not support write") } diff --git a/src/btreemap/node/v1.rs b/src/btreemap/node/v1.rs index 9bb5b411..8646324b 100644 --- a/src/btreemap/node/v1.rs +++ b/src/btreemap/node/v1.rs @@ -65,15 +65,14 @@ impl Node { // Load the entries. let mut keys_encoded_values = Vec::with_capacity(header.num_entries as usize); let mut offset = NodeHeader::size(); - let mut buf = Vec::with_capacity(max_key_size.max(max_value_size) as usize); + let mut buf = vec![]; for _ in 0..header.num_entries { // Read the key's size. let key_size = read_u32(memory, address + offset); offset += U32_SIZE; // Read the key. - buf.resize(key_size as usize, 0); - memory.read((address + offset).get(), &mut buf); + memory.read_to_vec((address + offset).get(), &mut buf, key_size as usize); offset += Bytes::from(max_key_size); let key = K::from_bytes(Cow::Borrowed(&buf)); // Values are loaded lazily. Store a reference and skip loading it. diff --git a/src/btreemap/node/v2.rs b/src/btreemap/node/v2.rs index 5f8699ab..b2bd3882 100644 --- a/src/btreemap/node/v2.rs +++ b/src/btreemap/node/v2.rs @@ -164,8 +164,7 @@ impl Node { }; // Load the key. - buf.resize(key_size as usize, 0); - reader.read(offset.get(), &mut buf); + reader.read_to_vec(offset.get(), &mut buf, key_size as usize); let key = K::from_bytes(Cow::Borrowed(&buf)); offset += Bytes::from(key_size); keys_encoded_values.push((key, Value::by_ref(Bytes::from(0usize)))); diff --git a/src/cell.rs b/src/cell.rs index 47b09a37..65eb64fd 100644 --- a/src/cell.rs +++ b/src/cell.rs @@ -1,6 +1,6 @@ //! A serializable value stored in the stable memory. use crate::storable::Storable; -use crate::{Memory, WASM_PAGE_SIZE}; +use crate::{Memory, MemoryExt, WASM_PAGE_SIZE}; use std::borrow::{Borrow, Cow}; use std::fmt; @@ -132,8 +132,8 @@ impl Cell { /// /// PRECONDITION: memory is large enough to contain the value. fn read_value(memory: &M, len: u32) -> T { - let mut buf = vec![0; len as usize]; - memory.read(HEADER_V1_SIZE, &mut buf); + let mut buf = vec![]; + memory.read_to_vec(HEADER_V1_SIZE, &mut buf, len as usize); T::from_bytes(Cow::Owned(buf)) } diff --git a/src/ic0_memory.rs b/src/ic0_memory.rs index 00078dd0..1e3eeb81 100644 --- a/src/ic0_memory.rs +++ b/src/ic0_memory.rs @@ -30,6 +30,12 @@ impl Memory for Ic0StableMemory { unsafe { stable64_read(dst.as_ptr() as u64, offset, dst.len() as u64) } } + #[inline] + unsafe fn read_unsafe(&self, offset: u64, dst: *mut u8, count: usize) { + // SAFETY: This is safe because of the ic0 api guarantees. + stable64_read(dst as u64, offset, count as u64); + } + #[inline] fn write(&self, offset: u64, src: &[u8]) { // SAFETY: This is safe because of the ic0 api guarantees. diff --git a/src/lib.rs b/src/lib.rs index efe8d1b1..e152f26f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,20 +46,62 @@ pub trait Memory { /// pages. (One WebAssembly page is 64Ki bytes.) fn size(&self) -> u64; - /// Tries to grow the memory by new_pages many pages containing + /// Tries to grow the memory by `pages` many pages containing /// zeroes. If successful, returns the previous size of the /// memory (in pages). Otherwise, returns -1. fn grow(&self, pages: u64) -> i64; - /// Copies the data referred to by offset out of the stable memory - /// and replaces the corresponding bytes in dst. + /// Copies the data referred to by `offset` out of the stable memory + /// and replaces the corresponding bytes in `dst`. fn read(&self, offset: u64, dst: &mut [u8]); - /// Copies the data referred to by src and replaces the + /// Copies `count` bytes of the data starting from `offset` out of the stable memory into the + /// buffer starting at `dst`. + /// + /// This method is an alternative to `read` which does not require initializing a buffer and may + /// therefore be faster. + /// + /// Callers must guarantee that + /// * it is valid to write `count` number of bytes starting from `dst`, + /// * `dst..dst + count` does not overlap with `self`. + /// + /// Implementations must guarantee that before the method returns, `count` number of bytes + /// starting from `dst` will be initialized. + #[inline] + unsafe fn read_unsafe(&self, offset: u64, dst: *mut u8, count: usize) { + // Initialize the buffer to make the slice valid. + std::ptr::write_bytes(dst, 0, count); + let slice = std::slice::from_raw_parts_mut(dst, count); + self.read(offset, slice) + } + + /// Copies the data referred to by `src` and replaces the /// corresponding segment starting at offset in the stable memory. fn write(&self, offset: u64, src: &[u8]); } +pub trait MemoryExt { + /// Copies `count` bytes of data starting from offset out of the stable memory into `dst`. + /// + /// Callers are allowed to pass vectors in state (e.g. empty vectors). + /// After the method returns, `dst.len() == count`. + /// This method is an alternative to `read` which does not require initializing a buffer and may + /// therefore be faster. + fn read_to_vec(&self, offset: u64, dst: &mut std::vec::Vec, count: usize); +} + +impl MemoryExt for M { + #[inline] + fn read_to_vec(&self, offset: u64, dst: &mut std::vec::Vec, count: usize) { + dst.clear(); + dst.reserve(count); + unsafe { + self.read_unsafe(offset, dst.as_mut_ptr(), count); + dst.set_len(count); + } + } +} + // A helper function that reads a single 32bit integer encoded as // little-endian from the specified memory at the specified offset. fn read_u32(m: &M, addr: Address) -> u32 { diff --git a/src/log.rs b/src/log.rs index 9a273f0a..46344158 100644 --- a/src/log.rs +++ b/src/log.rs @@ -54,7 +54,7 @@ //! ---------------------------------------- //! Unallocated space //! ``` -use crate::{read_u64, safe_write, write_u64, Address, GrowFailed, Memory, Storable}; +use crate::{read_u64, safe_write, write_u64, Address, GrowFailed, Memory, MemoryExt, Storable}; use std::borrow::Cow; use std::cell::RefCell; use std::fmt; @@ -331,8 +331,8 @@ impl Log { /// ignores the result. pub fn read_entry(&self, idx: u64, buf: &mut Vec) -> Result<(), NoSuchEntry> { let (offset, len) = self.entry_meta(idx).ok_or(NoSuchEntry)?; - buf.resize(len, 0); - self.data_memory.read(HEADER_OFFSET + offset, buf); + self.data_memory + .read_to_vec(HEADER_OFFSET + offset, buf, len); Ok(()) } diff --git a/src/memory_manager.rs b/src/memory_manager.rs index a84075be..bd50a46a 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -43,7 +43,7 @@ use crate::{ read_struct, types::{Address, Bytes}, - write, write_struct, Memory, WASM_PAGE_SIZE, + write, write_struct, Memory, MemoryExt, WASM_PAGE_SIZE, }; use std::cell::RefCell; use std::cmp::min; @@ -239,9 +239,9 @@ impl MemoryManagerInner { } // Check if the magic in the memory corresponds to this object. - let mut dst = vec![0; 3]; + let mut dst = [0; 3]; memory.read(0, &mut dst); - if dst != MAGIC { + if &dst != MAGIC { // No memory manager found. Create a new instance. MemoryManagerInner::new(memory, bucket_size_in_pages) } else { @@ -277,8 +277,12 @@ impl MemoryManagerInner { assert_eq!(&header.magic, MAGIC, "Bad magic."); assert_eq!(header.version, LAYOUT_VERSION, "Unsupported version."); - let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; - memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); + let mut buckets = vec![]; + memory.read_to_vec( + bucket_allocations_address(BucketId(0)).get(), + &mut buckets, + MAX_NUM_BUCKETS as usize, + ); let mut memory_buckets = BTreeMap::new(); for (bucket_idx, memory) in buckets.into_iter().enumerate() { diff --git a/src/vec_mem.rs b/src/vec_mem.rs index 4f1c5919..90b3bd98 100644 --- a/src/vec_mem.rs +++ b/src/vec_mem.rs @@ -39,6 +39,22 @@ impl Memory for RefCell> { dst.copy_from_slice(&self.borrow()[offset as usize..n as usize]); } + unsafe fn read_unsafe(&self, offset: u64, dst: *mut u8, count: usize) { + let n = offset + .checked_add(count as u64) + .expect("read: out of bounds"); + + if n as usize > self.borrow().len() { + panic!("read: out of bounds"); + } + + // SAFETY: + // - we just checked that self is long enough + // - the caller guarantees that there are at least count byte space after dst + // - we are copying bytes so the pointers are automatically aligned + std::ptr::copy(self.borrow().as_ptr().add(offset as usize), dst, count); + } + fn write(&self, offset: u64, src: &[u8]) { let n = offset .checked_add(src.len() as u64) @@ -61,6 +77,9 @@ impl Memory for Rc { fn read(&self, offset: u64, dst: &mut [u8]) { self.deref().read(offset, dst) } + unsafe fn read_unsafe(&self, offset: u64, dst: *mut u8, count: usize) { + self.deref().read_unsafe(offset, dst, count) + } fn write(&self, offset: u64, src: &[u8]) { self.deref().write(offset, src) } From 8100c162fafff47af8eda5ce5d84b968b6556fb7 Mon Sep 17 00:00:00 2001 From: David Frank Date: Thu, 21 Nov 2024 15:32:48 +0000 Subject: [PATCH 2/9] typo --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index e152f26f..6e305714 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,7 +83,7 @@ pub trait Memory { pub trait MemoryExt { /// Copies `count` bytes of data starting from offset out of the stable memory into `dst`. /// - /// Callers are allowed to pass vectors in state (e.g. empty vectors). + /// Callers are allowed to pass vectors in any state (e.g. empty vectors). /// After the method returns, `dst.len() == count`. /// This method is an alternative to `read` which does not require initializing a buffer and may /// therefore be faster. From 0ee50906475c8f255c222bb9e9bab91200de9b4c Mon Sep 17 00:00:00 2001 From: David Frank Date: Thu, 21 Nov 2024 15:38:34 +0000 Subject: [PATCH 3/9] Clippy --- src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6e305714..d0063cdb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -55,12 +55,14 @@ pub trait Memory { /// and replaces the corresponding bytes in `dst`. fn read(&self, offset: u64, dst: &mut [u8]); - /// Copies `count` bytes of the data starting from `offset` out of the stable memory into the - /// buffer starting at `dst`. + /// Copies `count` number of bytes of the data starting from `offset` out of the stable memory + /// into the buffer starting at `dst`. /// /// This method is an alternative to `read` which does not require initializing a buffer and may /// therefore be faster. /// + /// # Safety + /// /// Callers must guarantee that /// * it is valid to write `count` number of bytes starting from `dst`, /// * `dst..dst + count` does not overlap with `self`. From 1864bf0bee6fd08d018374fdeacfab49fca15d63 Mon Sep 17 00:00:00 2001 From: David Frank Date: Thu, 21 Nov 2024 23:15:40 +0100 Subject: [PATCH 4/9] comment Co-authored-by: Andriy Berestovskyy --- src/btreemap/node/io.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/btreemap/node/io.rs b/src/btreemap/node/io.rs index c69a6475..f73cb9bc 100644 --- a/src/btreemap/node/io.rs +++ b/src/btreemap/node/io.rs @@ -45,7 +45,7 @@ impl<'a, M: Memory> Memory for NodeReader<'a, M> { } in iter { // SAFETY: read_unsafe() is safe to call iff bytes_read + length <= count since the - // caller guarantees that we can write `count` number of bytes from `dst`. + // caller guarantees that we can write `count` number of bytes to `dst`. assert!(bytes_read + length.get() as usize <= count); if page_idx == 0 { self.memory.read_unsafe( From 6c2d21bc7f2497711a09891e46a6cc2cb3a03e50 Mon Sep 17 00:00:00 2001 From: David Frank Date: Thu, 21 Nov 2024 23:36:58 +0000 Subject: [PATCH 5/9] Change read_to_vec to normal function for consistency with the other functions --- canbench_results.yml | 212 ++++++++++++++++++++-------------------- src/base_vec.rs | 6 +- src/btreemap/node.rs | 13 ++- src/btreemap/node/v1.rs | 2 +- src/btreemap/node/v2.rs | 3 +- src/cell.rs | 4 +- src/lib.rs | 50 +++++----- src/log.rs | 5 +- src/memory_manager.rs | 9 +- 9 files changed, 153 insertions(+), 151 deletions(-) diff --git a/canbench_results.yml b/canbench_results.yml index abc8760e..cced3ade 100644 --- a/canbench_results.yml +++ b/canbench_results.yml @@ -1,595 +1,595 @@ benches: btreemap_get_blob_128_1024: total: - instructions: 826371552 + instructions: 819788807 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_128_1024_v2: total: - instructions: 912394951 + instructions: 894481632 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_16_1024: total: - instructions: 300853830 + instructions: 291484797 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_16_1024_v2: total: - instructions: 376695579 + instructions: 360074496 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_256_1024: total: - instructions: 1329458474 + instructions: 1321719369 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_256_1024_v2: total: - instructions: 1416464604 + instructions: 1399493032 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_32_1024: total: - instructions: 329605716 + instructions: 321831293 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_32_1024_v2: total: - instructions: 410560225 + instructions: 393309148 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_4_1024: total: - instructions: 180863991 + instructions: 180297441 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_4_1024_v2: total: - instructions: 269645388 + instructions: 258847572 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_512_1024: total: - instructions: 2338381852 + instructions: 2331297091 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_512_1024_v2: total: - instructions: 2421706283 + instructions: 2403763640 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_64_1024: total: - instructions: 573701035 + instructions: 571437015 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_64_1024_v2: total: - instructions: 662936241 + instructions: 644946866 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_1024: total: - instructions: 208888626 + instructions: 212420582 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_1024_v2: total: - instructions: 294374626 + instructions: 288886878 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_u64: total: - instructions: 196845655 + instructions: 196193945 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_u64_v2: total: - instructions: 298384613 + instructions: 287271984 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_blob_8: total: - instructions: 184588240 + instructions: 178105111 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_blob_8_v2: total: - instructions: 266224360 + instructions: 247512174 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_u64: total: - instructions: 185443402 + instructions: 178892725 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_u64_v2: total: - instructions: 272432249 + instructions: 254299677 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_insert_10mib_values: total: - instructions: 82723368 + instructions: 82050718 heap_increase: 0 stable_memory_increase: 32 scopes: {} btreemap_insert_blob_1024_128: total: - instructions: 4905425072 + instructions: 4878050549 heap_increase: 0 stable_memory_increase: 262 scopes: {} btreemap_insert_blob_1024_128_v2: total: - instructions: 4968538080 + instructions: 4959459872 heap_increase: 0 stable_memory_increase: 196 scopes: {} btreemap_insert_blob_1024_16: total: - instructions: 4893737868 + instructions: 4865943592 heap_increase: 0 stable_memory_increase: 241 scopes: {} btreemap_insert_blob_1024_16_v2: total: - instructions: 4947593670 + instructions: 4942887053 heap_increase: 0 stable_memory_increase: 181 scopes: {} btreemap_insert_blob_1024_256: total: - instructions: 4920366269 + instructions: 4904577059 heap_increase: 0 stable_memory_increase: 292 scopes: {} btreemap_insert_blob_1024_256_v2: total: - instructions: 4995926965 + instructions: 4988597163 heap_increase: 0 stable_memory_increase: 219 scopes: {} btreemap_insert_blob_1024_32: total: - instructions: 4905322013 + instructions: 4871649352 heap_increase: 0 stable_memory_increase: 239 scopes: {} btreemap_insert_blob_1024_32_v2: total: - instructions: 4965639927 + instructions: 4960073396 heap_increase: 0 stable_memory_increase: 180 scopes: {} btreemap_insert_blob_1024_4: total: - instructions: 4794955186 + instructions: 4769731635 heap_increase: 0 stable_memory_increase: 235 scopes: {} btreemap_insert_blob_1024_4_v2: total: - instructions: 4844178539 + instructions: 4838804816 heap_increase: 0 stable_memory_increase: 176 scopes: {} btreemap_insert_blob_1024_512: total: - instructions: 5009944961 + instructions: 4981176809 heap_increase: 0 stable_memory_increase: 348 scopes: {} btreemap_insert_blob_1024_512_v2: total: - instructions: 5084945715 + instructions: 5071050684 heap_increase: 0 stable_memory_increase: 261 scopes: {} btreemap_insert_blob_1024_64: total: - instructions: 4912581068 + instructions: 4889893021 heap_increase: 0 stable_memory_increase: 250 scopes: {} btreemap_insert_blob_1024_64_v2: total: - instructions: 4985803250 + instructions: 4979006046 heap_increase: 0 stable_memory_increase: 188 scopes: {} btreemap_insert_blob_1024_8: total: - instructions: 4856246096 + instructions: 4835672026 heap_increase: 0 stable_memory_increase: 237 scopes: {} btreemap_insert_blob_1024_8_v2: total: - instructions: 4933641074 + instructions: 4928203141 heap_increase: 0 stable_memory_increase: 178 scopes: {} btreemap_insert_blob_128_1024: total: - instructions: 1273170444 + instructions: 1254158857 heap_increase: 0 stable_memory_increase: 260 scopes: {} btreemap_insert_blob_128_1024_v2: total: - instructions: 1371510921 + instructions: 1344803156 heap_increase: 0 stable_memory_increase: 195 scopes: {} btreemap_insert_blob_16_1024: total: - instructions: 692954040 + instructions: 679931454 heap_increase: 0 stable_memory_increase: 215 scopes: {} btreemap_insert_blob_16_1024_v2: total: - instructions: 788011123 + instructions: 764214679 heap_increase: 0 stable_memory_increase: 161 scopes: {} btreemap_insert_blob_256_1024: total: - instructions: 1827822057 + instructions: 1808288914 heap_increase: 0 stable_memory_increase: 292 scopes: {} btreemap_insert_blob_256_1024_v2: total: - instructions: 1922268722 + instructions: 1896703939 heap_increase: 0 stable_memory_increase: 219 scopes: {} btreemap_insert_blob_32_1024: total: - instructions: 733238813 + instructions: 715174142 heap_increase: 0 stable_memory_increase: 230 scopes: {} btreemap_insert_blob_32_1024_v2: total: - instructions: 828693177 + instructions: 802944505 heap_increase: 0 stable_memory_increase: 173 scopes: {} btreemap_insert_blob_4_1024: total: - instructions: 494283887 + instructions: 485883652 heap_increase: 0 stable_memory_increase: 123 scopes: {} btreemap_insert_blob_4_1024_v2: total: - instructions: 592473788 + instructions: 574270275 heap_increase: 0 stable_memory_increase: 92 scopes: {} btreemap_insert_blob_512_1024: total: - instructions: 2918030428 + instructions: 2897522289 heap_increase: 0 stable_memory_increase: 351 scopes: {} btreemap_insert_blob_512_1024_v2: total: - instructions: 3003589075 + instructions: 2981803603 heap_increase: 0 stable_memory_increase: 263 scopes: {} btreemap_insert_blob_64_1024: total: - instructions: 999684307 + instructions: 983432897 heap_increase: 0 stable_memory_increase: 245 scopes: {} btreemap_insert_blob_64_1024_v2: total: - instructions: 1095310543 + instructions: 1069840736 heap_increase: 0 stable_memory_increase: 183 scopes: {} btreemap_insert_blob_8_1024: total: - instructions: 603879508 + instructions: 597682291 heap_increase: 0 stable_memory_increase: 183 scopes: {} btreemap_insert_blob_8_1024_v2: total: - instructions: 706921520 + instructions: 690616575 heap_increase: 0 stable_memory_increase: 138 scopes: {} btreemap_insert_blob_8_u64: total: - instructions: 328972005 + instructions: 322610289 heap_increase: 0 stable_memory_increase: 6 scopes: {} btreemap_insert_blob_8_u64_v2: total: - instructions: 440950328 + instructions: 424650180 heap_increase: 0 stable_memory_increase: 4 scopes: {} btreemap_insert_u64_blob_8: total: - instructions: 347528237 + instructions: 333762889 heap_increase: 0 stable_memory_increase: 7 scopes: {} btreemap_insert_u64_blob_8_v2: total: - instructions: 431734631 + instructions: 406563823 heap_increase: 0 stable_memory_increase: 5 scopes: {} btreemap_insert_u64_u64: total: - instructions: 353813009 + instructions: 339335885 heap_increase: 0 stable_memory_increase: 7 scopes: {} btreemap_insert_u64_u64_v2: total: - instructions: 440489004 + instructions: 415192316 heap_increase: 0 stable_memory_increase: 6 scopes: {} btreemap_iter_10mib_values: total: - instructions: 11431732 + instructions: 11386716 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_count_10mib_values: total: - instructions: 490726 + instructions: 474863 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_count_small_values: total: - instructions: 9637293 + instructions: 9436649 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_rev_10mib_values: total: - instructions: 11430523 + instructions: 11383689 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_rev_small_values: total: - instructions: 14754706 + instructions: 13557401 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_small_values: total: - instructions: 14713901 + instructions: 13564683 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_10mib_values: total: - instructions: 477180 + instructions: 463769 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_rev_10mib_values: total: - instructions: 479533 + instructions: 465166 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_rev_small_values: total: - instructions: 10009085 + instructions: 9803742 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_small_values: total: - instructions: 9773248 + instructions: 9571988 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_read_every_third_value_from_range: total: - instructions: 85060540 + instructions: 82602086 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_read_keys_from_range: total: - instructions: 85060540 + instructions: 82602086 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_128_1024: total: - instructions: 1536789523 + instructions: 1509965801 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_128_1024_v2: total: - instructions: 1678979750 + instructions: 1640532725 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_16_1024: total: - instructions: 810568836 + instructions: 788533979 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_16_1024_v2: total: - instructions: 947573091 + instructions: 913565235 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_256_1024: total: - instructions: 2182801039 + instructions: 2151972570 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_256_1024_v2: total: - instructions: 2316460898 + instructions: 2278321561 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_32_1024: total: - instructions: 876948171 + instructions: 851922918 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_32_1024_v2: total: - instructions: 1015025410 + instructions: 978847609 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_4_1024: total: - instructions: 484434371 + instructions: 472413549 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_4_1024_v2: total: - instructions: 600274873 + instructions: 577024150 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_512_1024: total: - instructions: 3529926666 + instructions: 3498163921 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_512_1024_v2: total: - instructions: 3653472751 + instructions: 3614587389 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_64_1024: total: - instructions: 1204953533 + instructions: 1179757290 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_64_1024_v2: total: - instructions: 1345064138 + instructions: 1307167707 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_1024: total: - instructions: 633622845 + instructions: 622872202 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_1024_v2: total: - instructions: 767528036 + instructions: 744254843 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_u64: total: - instructions: 434105759 + instructions: 422953606 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_u64_v2: total: - instructions: 585250331 + instructions: 561151074 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_blob_8: total: - instructions: 494615895 + instructions: 473536865 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_blob_8_v2: total: - instructions: 615668380 + instructions: 578924716 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_u64: total: - instructions: 508864341 + instructions: 486362140 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_u64_v2: total: - instructions: 639517433 + instructions: 601027136 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_10mib_values: total: - instructions: 11430320 + instructions: 11410497 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_rev_10mib_values: total: - instructions: 11429111 + instructions: 11408356 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_rev_small_values: total: - instructions: 14684044 + instructions: 14789365 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_small_values: total: - instructions: 14643239 + instructions: 14752643 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -613,43 +613,43 @@ benches: scopes: {} vec_get_blob_128: total: - instructions: 18554323 + instructions: 17779013 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_blob_16: total: - instructions: 8521818 + instructions: 7783358 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_blob_32: total: - instructions: 9170075 + instructions: 8410711 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_blob_4: total: - instructions: 5363359 + instructions: 4717509 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_blob_64: total: - instructions: 13553328 + instructions: 12783244 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_blob_8: total: - instructions: 6146813 + instructions: 5434935 heap_increase: 0 stable_memory_increase: 0 scopes: {} vec_get_u64: total: - instructions: 5540316 + instructions: 5240307 heap_increase: 0 stable_memory_increase: 0 scopes: {} diff --git a/src/base_vec.rs b/src/base_vec.rs index 814de59e..cc5b102f 100644 --- a/src/base_vec.rs +++ b/src/base_vec.rs @@ -33,8 +33,8 @@ //! bytes required to represent integers up to that max size. use crate::storable::{bounds, bytes_to_store_size_bounded}; use crate::{ - read_u32, read_u64, safe_write, write, write_u32, write_u64, Address, GrowFailed, Memory, - MemoryExt, Storable, + read_to_vec, read_u32, read_u64, safe_write, write, write_u32, write_u64, Address, GrowFailed, + Memory, Storable, }; use std::borrow::{Borrow, Cow}; use std::cmp::min; @@ -248,7 +248,7 @@ impl BaseVec { fn read_entry_to(&self, index: u64, buf: &mut Vec) { let offset = DATA_OFFSET + slot_size::() as u64 * index; let (data_offset, data_size) = self.read_entry_size(offset); - self.memory.read_to_vec(data_offset, buf, data_size); + read_to_vec(&self.memory, data_offset.into(), buf, data_size); } /// Sets the vector's length. diff --git a/src/btreemap/node.rs b/src/btreemap/node.rs index ef05037e..98288f64 100644 --- a/src/btreemap/node.rs +++ b/src/btreemap/node.rs @@ -1,9 +1,9 @@ use crate::{ btreemap::Allocator, - read_struct, read_u32, read_u64, + read_struct, read_to_vec, read_u32, read_u64, storable::Storable, types::{Address, Bytes}, - write, write_struct, write_u32, Memory, MemoryExt, + write, write_struct, write_u32, Memory, }; use std::borrow::{Borrow, Cow}; use std::cell::OnceCell; @@ -190,7 +190,7 @@ impl Node { value.take_or_load(|offset| self.load_value_from_memory(offset, memory)) } - /// Loads a value from stable memory at the given offset. + /// Loads a value from stable memory at the given offset of this node. fn load_value_from_memory(&self, offset: Bytes, memory: &M) -> Vec { let reader = NodeReader { address: self.address, @@ -201,7 +201,12 @@ impl Node { let value_len = read_u32(&reader, Address::from(offset.get())) as usize; let mut bytes = vec![]; - reader.read_to_vec((offset + U32_SIZE).get(), &mut bytes, value_len); + read_to_vec( + &reader, + Address::from((offset + U32_SIZE).get()), + &mut bytes, + value_len, + ); bytes } diff --git a/src/btreemap/node/v1.rs b/src/btreemap/node/v1.rs index 8646324b..215cecf0 100644 --- a/src/btreemap/node/v1.rs +++ b/src/btreemap/node/v1.rs @@ -72,7 +72,7 @@ impl Node { offset += U32_SIZE; // Read the key. - memory.read_to_vec((address + offset).get(), &mut buf, key_size as usize); + read_to_vec(memory, address + offset, &mut buf, key_size as usize); offset += Bytes::from(max_key_size); let key = K::from_bytes(Cow::Borrowed(&buf)); // Values are loaded lazily. Store a reference and skip loading it. diff --git a/src/btreemap/node/v2.rs b/src/btreemap/node/v2.rs index b2bd3882..c798d95e 100644 --- a/src/btreemap/node/v2.rs +++ b/src/btreemap/node/v2.rs @@ -141,6 +141,7 @@ impl Node { let mut children = vec![]; if node_type == NodeType::Internal { // The number of children is equal to the number of entries + 1. + children.reserve(num_entries + 1); for _ in 0..num_entries + 1 { let child = Address::from(read_u64(&reader, offset)); offset += Address::size(); @@ -164,7 +165,7 @@ impl Node { }; // Load the key. - reader.read_to_vec(offset.get(), &mut buf, key_size as usize); + read_to_vec(&reader, offset, &mut buf, key_size as usize); let key = K::from_bytes(Cow::Borrowed(&buf)); offset += Bytes::from(key_size); keys_encoded_values.push((key, Value::by_ref(Bytes::from(0usize)))); diff --git a/src/cell.rs b/src/cell.rs index 65eb64fd..33f672dc 100644 --- a/src/cell.rs +++ b/src/cell.rs @@ -1,6 +1,6 @@ //! A serializable value stored in the stable memory. use crate::storable::Storable; -use crate::{Memory, MemoryExt, WASM_PAGE_SIZE}; +use crate::{read_to_vec, Memory, WASM_PAGE_SIZE}; use std::borrow::{Borrow, Cow}; use std::fmt; @@ -133,7 +133,7 @@ impl Cell { /// PRECONDITION: memory is large enough to contain the value. fn read_value(memory: &M, len: u32) -> T { let mut buf = vec![]; - memory.read_to_vec(HEADER_V1_SIZE, &mut buf, len as usize); + read_to_vec(memory, HEADER_V1_SIZE.into(), &mut buf, len as usize); T::from_bytes(Cow::Owned(buf)) } diff --git a/src/lib.rs b/src/lib.rs index d0063cdb..59fbebd9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,7 @@ pub use file_mem::FileMemory; pub use ic0_memory::Ic0StableMemory; use std::error; use std::fmt::{Display, Formatter}; +use std::mem::MaybeUninit; pub use storable::Storable; use types::Address; pub use vec_mem::VectorMemory; @@ -82,50 +83,45 @@ pub trait Memory { fn write(&self, offset: u64, src: &[u8]); } -pub trait MemoryExt { - /// Copies `count` bytes of data starting from offset out of the stable memory into `dst`. - /// - /// Callers are allowed to pass vectors in any state (e.g. empty vectors). - /// After the method returns, `dst.len() == count`. - /// This method is an alternative to `read` which does not require initializing a buffer and may - /// therefore be faster. - fn read_to_vec(&self, offset: u64, dst: &mut std::vec::Vec, count: usize); -} - -impl MemoryExt for M { - #[inline] - fn read_to_vec(&self, offset: u64, dst: &mut std::vec::Vec, count: usize) { - dst.clear(); - dst.reserve(count); - unsafe { - self.read_unsafe(offset, dst.as_mut_ptr(), count); - dst.set_len(count); - } +/// Copies `count` bytes of data starting from `addr` out of the stable memory into `dst`. +/// +/// Callers are allowed to pass vectors in any state (e.g. empty vectors). +/// After the method returns, `dst.len() == count`. +/// This method is an alternative to `read` which does not require initializing a buffer and may +/// therefore be faster. +#[inline] +fn read_to_vec(m: &M, addr: Address, dst: &mut std::vec::Vec, count: usize) { + dst.clear(); + dst.reserve(count); + unsafe { + m.read_unsafe(addr.get(), dst.as_mut_ptr(), count); + // SAFETY: read_unsafe guarantees to initialize the first `count` bytes + dst.set_len(count); } } -// A helper function that reads a single 32bit integer encoded as -// little-endian from the specified memory at the specified offset. +/// A helper function that reads a single 32bit integer encoded as +/// little-endian from the specified memory at the specified offset. fn read_u32(m: &M, addr: Address) -> u32 { let mut buf: [u8; 4] = [0; 4]; m.read(addr.get(), &mut buf); u32::from_le_bytes(buf) } -// A helper function that reads a single 64bit integer encoded as -// little-endian from the specified memory at the specified offset. +/// A helper function that reads a single 64bit integer encoded as +/// little-endian from the specified memory at the specified offset. fn read_u64(m: &M, addr: Address) -> u64 { let mut buf: [u8; 8] = [0; 8]; m.read(addr.get(), &mut buf); u64::from_le_bytes(buf) } -// Writes a single 32-bit integer encoded as little-endian. +/// Writes a single 32-bit integer encoded as little-endian. fn write_u32(m: &M, addr: Address, val: u32) { write(m, addr.get(), &val.to_le_bytes()); } -// Writes a single 64-bit integer encoded as little-endian. +/// Writes a single 64-bit integer encoded as little-endian. fn write_u64(m: &M, addr: Address, val: u64) { write(m, addr.get(), &val.to_le_bytes()); } @@ -192,7 +188,7 @@ fn write(memory: &M, offset: u64, bytes: &[u8]) { } } -// Reads a struct from memory. +/// Reads a struct from memory. fn read_struct(addr: Address, memory: &M) -> T { let mut t: T = unsafe { core::mem::zeroed() }; let t_slice = unsafe { @@ -202,7 +198,7 @@ fn read_struct(addr: Address, memory: &M) -> T { t } -// Writes a struct to memory. +/// Writes a struct to memory. fn write_struct(t: &T, addr: Address, memory: &M) { let slice = unsafe { core::slice::from_raw_parts(t as *const _ as *const u8, core::mem::size_of::()) diff --git a/src/log.rs b/src/log.rs index 46344158..f6d1d284 100644 --- a/src/log.rs +++ b/src/log.rs @@ -54,7 +54,7 @@ //! ---------------------------------------- //! Unallocated space //! ``` -use crate::{read_u64, safe_write, write_u64, Address, GrowFailed, Memory, MemoryExt, Storable}; +use crate::{read_to_vec, read_u64, safe_write, write_u64, Address, GrowFailed, Memory, Storable}; use std::borrow::Cow; use std::cell::RefCell; use std::fmt; @@ -331,8 +331,7 @@ impl Log { /// ignores the result. pub fn read_entry(&self, idx: u64, buf: &mut Vec) -> Result<(), NoSuchEntry> { let (offset, len) = self.entry_meta(idx).ok_or(NoSuchEntry)?; - self.data_memory - .read_to_vec(HEADER_OFFSET + offset, buf, len); + read_to_vec(&self.data_memory, (HEADER_OFFSET + offset).into(), buf, len); Ok(()) } diff --git a/src/memory_manager.rs b/src/memory_manager.rs index bd50a46a..fd30182a 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -41,9 +41,9 @@ //! assert_eq!(bytes, vec![4, 5, 6]); //! ``` use crate::{ - read_struct, + read_struct, read_to_vec, types::{Address, Bytes}, - write, write_struct, Memory, MemoryExt, WASM_PAGE_SIZE, + write, write_struct, Memory, WASM_PAGE_SIZE, }; use std::cell::RefCell; use std::cmp::min; @@ -278,8 +278,9 @@ impl MemoryManagerInner { assert_eq!(header.version, LAYOUT_VERSION, "Unsupported version."); let mut buckets = vec![]; - memory.read_to_vec( - bucket_allocations_address(BucketId(0)).get(), + read_to_vec( + &memory, + bucket_allocations_address(BucketId(0)), &mut buckets, MAX_NUM_BUCKETS as usize, ); From 780d1cbb48980349e0616390e06722ff5f402272 Mon Sep 17 00:00:00 2001 From: David Frank Date: Thu, 21 Nov 2024 23:41:22 +0000 Subject: [PATCH 6/9] Reimplemented read_struct using read_unsafe The previous implementation wasn't fully safe according to Rust standards because - T was initialized with zeros which may be illegal depending on T - t and t_slice were aliasing the same mutable memory --- src/lib.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 59fbebd9..b7114319 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -190,12 +190,15 @@ fn write(memory: &M, offset: u64, bytes: &[u8]) { /// Reads a struct from memory. fn read_struct(addr: Address, memory: &M) -> T { - let mut t: T = unsafe { core::mem::zeroed() }; - let t_slice = unsafe { - core::slice::from_raw_parts_mut(&mut t as *mut _ as *mut u8, core::mem::size_of::()) - }; - memory.read(addr.get(), t_slice); - t + let mut value = MaybeUninit::::uninit(); + unsafe { + memory.read_unsafe( + addr.get(), + value.as_mut_ptr() as *mut u8, + core::mem::size_of::(), + ); + value.assume_init() + } } /// Writes a struct to memory. From a3765db5f58a4e69a7aaaf7992b9224d1db900d0 Mon Sep 17 00:00:00 2001 From: David Frank Date: Fri, 22 Nov 2024 00:05:32 +0000 Subject: [PATCH 7/9] Add tests --- src/tests.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/tests.rs b/src/tests.rs index 3d138d68..25f8fa33 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -66,3 +66,37 @@ fn should_fail_to_recover_memory_from_memory_manager_if_memory_is_in_use() { let recovered_memory = memory_manager.into_memory(); assert!(recovered_memory.is_none()); } + +#[test] +fn test_read_to_vec_roundtrip() { + let memory = DefaultMemoryImpl::default(); + memory.grow(1); + memory.write(0, &[5, 6, 7, 8, 9]); + + let mut out = vec![]; + read_to_vec(&memory, Address::from(0), &mut out, 5); + assert_eq!(out, vec![5, 6, 7, 8, 9]); +} + +#[test] +fn test_read_write_struct_roundtrip() { + #[derive(Eq, PartialEq, Debug)] + struct Foo { + a: i32, + b: [char; 5], + } + + let foo = Foo { + a: 42, + b: ['a', 'b', 'c', 'd', 'e'], + }; + + let memory = DefaultMemoryImpl::default(); + memory.grow(1); + write_struct(&foo, Address::from(3), &memory); + + assert_eq!( + read_struct::(Address::from(3), &memory), + foo + ) +} From c99d2e11550d780dfd6aabd2e1e2358232cfcaca Mon Sep 17 00:00:00 2001 From: David Frank Date: Fri, 22 Nov 2024 08:45:35 +0000 Subject: [PATCH 8/9] Use reserve_exact which has better performance in our benchmarks --- canbench_results.yml | 198 ++++++++++++++++++++-------------------- src/btreemap/node/v1.rs | 2 +- src/btreemap/node/v2.rs | 2 +- src/lib.rs | 2 +- 4 files changed, 102 insertions(+), 102 deletions(-) diff --git a/canbench_results.yml b/canbench_results.yml index cced3ade..d4a0e687 100644 --- a/canbench_results.yml +++ b/canbench_results.yml @@ -1,595 +1,595 @@ benches: btreemap_get_blob_128_1024: total: - instructions: 819788807 + instructions: 816917041 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_128_1024_v2: total: - instructions: 894481632 + instructions: 891561192 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_16_1024: total: - instructions: 291484797 + instructions: 288610553 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_16_1024_v2: total: - instructions: 360074496 + instructions: 357151536 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_256_1024: total: - instructions: 1321719369 + instructions: 1318848488 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_256_1024_v2: total: - instructions: 1399493032 + instructions: 1396573492 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_32_1024: total: - instructions: 321831293 + instructions: 318958937 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_32_1024_v2: total: - instructions: 393309148 + instructions: 390388108 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_4_1024: total: - instructions: 180297441 + instructions: 178025469 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_4_1024_v2: total: - instructions: 258847572 + instructions: 256537092 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_512_1024: total: - instructions: 2331297091 + instructions: 2328426328 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_512_1024_v2: total: - instructions: 2403763640 + instructions: 2400844220 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_64_1024: total: - instructions: 571437015 + instructions: 568565131 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_64_1024_v2: total: - instructions: 644946866 + instructions: 642026306 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_1024: total: - instructions: 212420582 + instructions: 209549111 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_1024_v2: total: - instructions: 288886878 + instructions: 285966738 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_u64: total: - instructions: 196193945 + instructions: 193326958 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_blob_8_u64_v2: total: - instructions: 287271984 + instructions: 284356404 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_blob_8: total: - instructions: 178105111 + instructions: 175234112 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_blob_8_v2: total: - instructions: 247512174 + instructions: 244592514 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_u64: total: - instructions: 178892725 + instructions: 176021844 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_get_u64_u64_v2: total: - instructions: 254299677 + instructions: 251380137 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_insert_10mib_values: total: - instructions: 82050718 + instructions: 82015558 heap_increase: 0 stable_memory_increase: 32 scopes: {} btreemap_insert_blob_1024_128: total: - instructions: 4878050549 + instructions: 4875355783 heap_increase: 0 stable_memory_increase: 262 scopes: {} btreemap_insert_blob_1024_128_v2: total: - instructions: 4959459872 + instructions: 4956719432 heap_increase: 0 stable_memory_increase: 196 scopes: {} btreemap_insert_blob_1024_16: total: - instructions: 4865943592 + instructions: 4863260567 heap_increase: 0 stable_memory_increase: 241 scopes: {} btreemap_insert_blob_1024_16_v2: total: - instructions: 4942887053 + instructions: 4940158553 heap_increase: 0 stable_memory_increase: 181 scopes: {} btreemap_insert_blob_1024_256: total: - instructions: 4904577059 + instructions: 4901893503 heap_increase: 0 stable_memory_increase: 292 scopes: {} btreemap_insert_blob_1024_256_v2: total: - instructions: 4988597163 + instructions: 4985868123 heap_increase: 0 stable_memory_increase: 219 scopes: {} btreemap_insert_blob_1024_32: total: - instructions: 4871649352 + instructions: 4869006447 heap_increase: 0 stable_memory_increase: 239 scopes: {} btreemap_insert_blob_1024_32_v2: total: - instructions: 4960073396 + instructions: 4957385696 heap_increase: 0 stable_memory_increase: 180 scopes: {} btreemap_insert_blob_1024_4: total: - instructions: 4769731635 + instructions: 4767021647 heap_increase: 0 stable_memory_increase: 235 scopes: {} btreemap_insert_blob_1024_4_v2: total: - instructions: 4838804816 + instructions: 4836048896 heap_increase: 0 stable_memory_increase: 176 scopes: {} btreemap_insert_blob_1024_512: total: - instructions: 4981176809 + instructions: 4978502221 heap_increase: 0 stable_memory_increase: 348 scopes: {} btreemap_insert_blob_1024_512_v2: total: - instructions: 5071050684 + instructions: 5068330764 heap_increase: 0 stable_memory_increase: 261 scopes: {} btreemap_insert_blob_1024_64: total: - instructions: 4889893021 + instructions: 4887214834 heap_increase: 0 stable_memory_increase: 250 scopes: {} btreemap_insert_blob_1024_64_v2: total: - instructions: 4979006046 + instructions: 4976282466 heap_increase: 0 stable_memory_increase: 188 scopes: {} btreemap_insert_blob_1024_8: total: - instructions: 4835672026 + instructions: 4832986464 heap_increase: 0 stable_memory_increase: 237 scopes: {} btreemap_insert_blob_1024_8_v2: total: - instructions: 4928203141 + instructions: 4925472061 heap_increase: 0 stable_memory_increase: 178 scopes: {} btreemap_insert_blob_128_1024: total: - instructions: 1254158857 + instructions: 1251468811 heap_increase: 0 stable_memory_increase: 260 scopes: {} btreemap_insert_blob_128_1024_v2: total: - instructions: 1344803156 + instructions: 1342067516 heap_increase: 0 stable_memory_increase: 195 scopes: {} btreemap_insert_blob_16_1024: total: - instructions: 679931454 + instructions: 677272206 heap_increase: 0 stable_memory_increase: 215 scopes: {} btreemap_insert_blob_16_1024_v2: total: - instructions: 764214679 + instructions: 761510359 heap_increase: 0 stable_memory_increase: 161 scopes: {} btreemap_insert_blob_256_1024: total: - instructions: 1808288914 + instructions: 1805606715 heap_increase: 0 stable_memory_increase: 292 scopes: {} btreemap_insert_blob_256_1024_v2: total: - instructions: 1896703939 + instructions: 1893976279 heap_increase: 0 stable_memory_increase: 219 scopes: {} btreemap_insert_blob_32_1024: total: - instructions: 715174142 + instructions: 712489642 heap_increase: 0 stable_memory_increase: 230 scopes: {} btreemap_insert_blob_32_1024_v2: total: - instructions: 802944505 + instructions: 800214505 heap_increase: 0 stable_memory_increase: 173 scopes: {} btreemap_insert_blob_4_1024: total: - instructions: 485883652 + instructions: 483550320 heap_increase: 0 stable_memory_increase: 123 scopes: {} btreemap_insert_blob_4_1024_v2: total: - instructions: 574270275 + instructions: 571897395 heap_increase: 0 stable_memory_increase: 92 scopes: {} btreemap_insert_blob_512_1024: total: - instructions: 2897522289 + instructions: 2894814130 heap_increase: 0 stable_memory_increase: 351 scopes: {} btreemap_insert_blob_512_1024_v2: total: - instructions: 2981803603 + instructions: 2979049543 heap_increase: 0 stable_memory_increase: 263 scopes: {} btreemap_insert_blob_64_1024: total: - instructions: 983432897 + instructions: 980721257 heap_increase: 0 stable_memory_increase: 245 scopes: {} btreemap_insert_blob_64_1024_v2: total: - instructions: 1069840736 + instructions: 1067083136 heap_increase: 0 stable_memory_increase: 183 scopes: {} btreemap_insert_blob_8_1024: total: - instructions: 597682291 + instructions: 595149775 heap_increase: 0 stable_memory_increase: 183 scopes: {} btreemap_insert_blob_8_1024_v2: total: - instructions: 690616575 + instructions: 688041135 heap_increase: 0 stable_memory_increase: 138 scopes: {} btreemap_insert_blob_8_u64: total: - instructions: 322610289 + instructions: 320065029 heap_increase: 0 stable_memory_increase: 6 scopes: {} btreemap_insert_blob_8_u64_v2: total: - instructions: 424650180 + instructions: 422061780 heap_increase: 0 stable_memory_increase: 4 scopes: {} btreemap_insert_u64_blob_8: total: - instructions: 333762889 + instructions: 331086826 heap_increase: 0 stable_memory_increase: 7 scopes: {} btreemap_insert_u64_blob_8_v2: total: - instructions: 406563823 + instructions: 403842403 heap_increase: 0 stable_memory_increase: 5 scopes: {} btreemap_insert_u64_u64: total: - instructions: 339335885 + instructions: 336638169 heap_increase: 0 stable_memory_increase: 7 scopes: {} btreemap_insert_u64_u64_v2: total: - instructions: 415192316 + instructions: 412448876 heap_increase: 0 stable_memory_increase: 6 scopes: {} btreemap_iter_10mib_values: total: - instructions: 11386716 + instructions: 11384376 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_count_10mib_values: total: - instructions: 474863 + instructions: 472523 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_count_small_values: total: - instructions: 9436649 + instructions: 9316829 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_rev_10mib_values: total: - instructions: 11383689 + instructions: 11381349 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_rev_small_values: total: - instructions: 13557401 + instructions: 13437581 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_iter_small_values: total: - instructions: 13564683 + instructions: 13444863 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_10mib_values: total: - instructions: 463769 + instructions: 461429 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_rev_10mib_values: total: - instructions: 465166 + instructions: 462826 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_rev_small_values: total: - instructions: 9803742 + instructions: 9683922 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_keys_small_values: total: - instructions: 9571988 + instructions: 9452168 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_read_every_third_value_from_range: total: - instructions: 82602086 + instructions: 82482266 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_read_keys_from_range: total: - instructions: 82602086 + instructions: 82482266 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_128_1024: total: - instructions: 1509965801 + instructions: 1506697673 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_128_1024_v2: total: - instructions: 1640532725 + instructions: 1637209205 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_16_1024: total: - instructions: 788533979 + instructions: 785362788 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_16_1024_v2: total: - instructions: 913565235 + instructions: 910340295 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_256_1024: total: - instructions: 2151972570 + instructions: 2148746627 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_256_1024_v2: total: - instructions: 2278321561 + instructions: 2275040941 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_32_1024: total: - instructions: 851922918 + instructions: 848733142 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_32_1024_v2: total: - instructions: 978847609 + instructions: 975603769 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_4_1024: total: - instructions: 472413549 + instructions: 469755304 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_4_1024_v2: total: - instructions: 577024150 + instructions: 574320850 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_512_1024: total: - instructions: 3498163921 + instructions: 3494883108 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_512_1024_v2: total: - instructions: 3614587389 + instructions: 3611250969 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_64_1024: total: - instructions: 1179757290 + instructions: 1176535123 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_64_1024_v2: total: - instructions: 1307167707 + instructions: 1303890927 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_1024: total: - instructions: 622872202 + instructions: 619861550 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_1024_v2: total: - instructions: 744254843 + instructions: 741193163 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_u64: total: - instructions: 422953606 + instructions: 419913395 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_blob_8_u64_v2: total: - instructions: 561151074 + instructions: 558059334 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_blob_8: total: - instructions: 473536865 + instructions: 470317117 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_blob_8_v2: total: - instructions: 578924716 + instructions: 575650396 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_u64: total: - instructions: 486362140 + instructions: 483160741 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_remove_u64_u64_v2: total: - instructions: 601027136 + instructions: 597771476 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_10mib_values: total: - instructions: 11410497 + instructions: 11408157 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_rev_10mib_values: total: - instructions: 11408356 + instructions: 11406016 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_rev_small_values: total: - instructions: 14789365 + instructions: 14669545 heap_increase: 0 stable_memory_increase: 0 scopes: {} btreemap_values_small_values: total: - instructions: 14752643 + instructions: 14632823 heap_increase: 0 stable_memory_increase: 0 scopes: {} diff --git a/src/btreemap/node/v1.rs b/src/btreemap/node/v1.rs index 215cecf0..2dc02857 100644 --- a/src/btreemap/node/v1.rs +++ b/src/btreemap/node/v1.rs @@ -85,7 +85,7 @@ impl Node { let mut children = vec![]; if header.node_type == INTERNAL_NODE_TYPE { // The number of children is equal to the number of entries + 1. - children.reserve(header.num_entries as usize + 1); + children.reserve_exact(header.num_entries as usize + 1); for _ in 0..header.num_entries + 1 { let child = Address::from(read_u64(memory, address + offset)); offset += Address::size(); diff --git a/src/btreemap/node/v2.rs b/src/btreemap/node/v2.rs index c798d95e..f559d777 100644 --- a/src/btreemap/node/v2.rs +++ b/src/btreemap/node/v2.rs @@ -141,7 +141,7 @@ impl Node { let mut children = vec![]; if node_type == NodeType::Internal { // The number of children is equal to the number of entries + 1. - children.reserve(num_entries + 1); + children.reserve_exact(num_entries + 1); for _ in 0..num_entries + 1 { let child = Address::from(read_u64(&reader, offset)); offset += Address::size(); diff --git a/src/lib.rs b/src/lib.rs index b7114319..05be22a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -92,7 +92,7 @@ pub trait Memory { #[inline] fn read_to_vec(m: &M, addr: Address, dst: &mut std::vec::Vec, count: usize) { dst.clear(); - dst.reserve(count); + dst.reserve_exact(count); unsafe { m.read_unsafe(addr.get(), dst.as_mut_ptr(), count); // SAFETY: read_unsafe guarantees to initialize the first `count` bytes From d378e2fb5335f233281ad4a03e8f49c880ef4296 Mon Sep 17 00:00:00 2001 From: David Frank Date: Fri, 22 Nov 2024 11:46:37 +0000 Subject: [PATCH 9/9] Add VectorMemory tests --- src/tests.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/tests.rs b/src/tests.rs index 25f8fa33..faa01d33 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -100,3 +100,27 @@ fn test_read_write_struct_roundtrip() { foo ) } + +#[test] +fn test_vector_memory_read() { + let memory = VectorMemory::default(); + memory.grow(1); + memory.write(1, &[4, 6, 8]); + + { + let mut buffer = [0; 3]; + memory.read(1, &mut buffer[..]); + assert_eq!(buffer, [4, 6, 8]); + } + + { + let mut buffer = std::vec::Vec::with_capacity(3); + unsafe { + let ptr = buffer.as_mut_ptr(); + memory.read_unsafe(1, buffer.as_mut_ptr(), 3); + assert_eq!(std::ptr::read(ptr), 4); + assert_eq!(std::ptr::read(ptr.add(1)), 6); + assert_eq!(std::ptr::read(ptr.add(2)), 8); + } + } +}