From 659966188e6e01ce45ba1571a41ced23236b8f78 Mon Sep 17 00:00:00 2001 From: Andreas Garnaes Date: Sun, 11 Jun 2023 11:31:41 +0200 Subject: [PATCH 1/6] Add serializers to Bitmap module --- croaring/src/bitmap/imp.rs | 63 ++------------ croaring/src/bitmap/mod.rs | 2 + croaring/src/bitmap/serialization.rs | 121 +++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 57 deletions(-) create mode 100644 croaring/src/bitmap/serialization.rs diff --git a/croaring/src/bitmap/imp.rs b/croaring/src/bitmap/imp.rs index 9e6fd51..0552164 100644 --- a/croaring/src/bitmap/imp.rs +++ b/croaring/src/bitmap/imp.rs @@ -1,7 +1,6 @@ use crate::Bitset; use ffi::roaring_bitmap_t; use std::convert::TryInto; -use std::ffi::c_char; use std::mem; use std::ops::{Bound, RangeBounds}; @@ -10,7 +9,7 @@ use super::{Bitmap, Statistics}; impl Bitmap { #[inline] #[allow(clippy::assertions_on_constants)] - unsafe fn take_heap(p: *mut roaring_bitmap_t) -> Self { + pub(crate) unsafe fn take_heap(p: *mut roaring_bitmap_t) -> Self { // Based heavily on the `roaring.hh` cpp header from croaring assert!(!p.is_null()); @@ -738,14 +737,14 @@ impl Bitmap { #[inline] #[doc(alias = "roaring_bitmap_portable_size_in_bytes")] pub fn get_serialized_size_in_bytes(&self) -> usize { - unsafe { ffi::roaring_bitmap_portable_size_in_bytes(&self.bitmap) } + super::PortableSerializer::get_serialized_size_in_bytes(&self) } /// Computes the serialized size in bytes of the Bitmap for the frozen format. #[inline] #[doc(alias = "roaring_bitmap_frozen_size_in_bytes")] pub fn get_frozen_serialized_size_in_bytes(&self) -> usize { - unsafe { ffi::roaring_bitmap_frozen_size_in_bytes(&self.bitmap) } + super::FrozenSerializer::get_serialized_size_in_bytes(&self) } /// Serializes a bitmap to a slice of bytes. @@ -794,20 +793,7 @@ impl Bitmap { #[inline] #[doc(alias = "roaring_bitmap_portable_serialize")] pub fn serialize_into<'a>(&self, dst: &'a mut Vec) -> &'a [u8] { - let len = self.get_serialized_size_in_bytes(); - - dst.reserve(len); - let total_len = dst.len().checked_add(len).unwrap(); - - unsafe { - ffi::roaring_bitmap_portable_serialize( - &self.bitmap, - dst.spare_capacity_mut().as_mut_ptr().cast::(), - ); - dst.set_len(total_len); - } - - dst + super::PortableSerializer::serialize_into(self, dst) } /// Serialize into the "frozen" format @@ -816,33 +802,7 @@ impl Bitmap { /// This means the returned slice may not start exactly at the beginning of the passed Vec #[doc(alias = "roaring_bitmap_frozen_serialize")] pub fn serialize_frozen_into<'a>(&self, dst: &'a mut Vec) -> &'a [u8] { - const REQUIRED_ALIGNMENT: usize = 32; - let len = self.get_frozen_serialized_size_in_bytes(); - - let offset = dst.len(); - // Need to be able to add up to 31 extra bytes to align to 32 bytes - dst.reserve(len.checked_add(REQUIRED_ALIGNMENT - 1).unwrap()); - - let extra_offset = match (dst.as_ptr() as usize) % REQUIRED_ALIGNMENT { - 0 => 0, - r => REQUIRED_ALIGNMENT - r, - }; - let offset = offset.checked_add(extra_offset).unwrap(); - let total_len = offset.checked_add(len).unwrap(); - debug_assert!(dst.capacity() >= total_len); - - // we must initialize up to offset - dst.resize(offset, 0); - - unsafe { - ffi::roaring_bitmap_frozen_serialize( - &self.bitmap, - dst.as_mut_ptr().add(offset).cast::(), - ); - dst.set_len(total_len); - } - - &dst[offset..total_len] + super::FrozenSerializer::serialize_into(self, dst) } /// Given a serialized bitmap as slice of bytes returns a bitmap instance. @@ -868,18 +828,7 @@ impl Bitmap { #[inline] #[doc(alias = "roaring_bitmap_portable_deserialize_safe")] pub fn try_deserialize(buffer: &[u8]) -> Option { - unsafe { - let bitmap = ffi::roaring_bitmap_portable_deserialize_safe( - buffer.as_ptr() as *const c_char, - buffer.len(), - ); - - if !bitmap.is_null() { - Some(Self::take_heap(bitmap)) - } else { - None - } - } + super::PortableSerializer::try_deserialize(buffer) } /// Given a serialized bitmap as slice of bytes returns a bitmap instance. diff --git a/croaring/src/bitmap/mod.rs b/croaring/src/bitmap/mod.rs index b50cbda..97541a2 100644 --- a/croaring/src/bitmap/mod.rs +++ b/croaring/src/bitmap/mod.rs @@ -86,7 +86,9 @@ mod imp; mod iter; mod lazy; mod ops; +mod serialization; mod view; pub use self::iter::BitmapIterator; pub use self::lazy::LazyBitmap; +pub use self::serialization::{FrozenSerializer, NativeSerializer, PortableSerializer}; diff --git a/croaring/src/bitmap/serialization.rs b/croaring/src/bitmap/serialization.rs new file mode 100644 index 0000000..8719eed --- /dev/null +++ b/croaring/src/bitmap/serialization.rs @@ -0,0 +1,121 @@ +use super::Bitmap; + +use std::ffi::{c_char, c_void}; + +pub struct PortableSerializer {} + +impl PortableSerializer { + pub fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { + let len = Self::get_serialized_size_in_bytes(bitmap); + + dst.reserve(len); + let total_len = dst.len().checked_add(len).unwrap(); + + unsafe { + ffi::roaring_bitmap_portable_serialize( + &bitmap.bitmap, + dst.spare_capacity_mut().as_mut_ptr().cast::(), + ); + dst.set_len(total_len); + } + + dst + } + + pub fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { + unsafe { ffi::roaring_bitmap_portable_size_in_bytes(&bitmap.bitmap) } + } + + pub fn try_deserialize(buffer: &[u8]) -> Option { + unsafe { + let bitmap = ffi::roaring_bitmap_portable_deserialize_safe( + buffer.as_ptr() as *const c_char, + buffer.len(), + ); + + if !bitmap.is_null() { + Some(Bitmap::take_heap(bitmap)) + } else { + None + } + } + } +} + +pub struct NativeSerializer {} + +impl NativeSerializer { + pub fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { + let len = Self::get_serialized_size_in_bytes(bitmap); + + dst.reserve(len); + let total_len = dst.len().checked_add(len).unwrap(); + + unsafe { + ffi::roaring_bitmap_serialize( + &bitmap.bitmap, + dst.spare_capacity_mut().as_mut_ptr().cast::(), + ); + dst.set_len(total_len); + } + + dst + } + + pub fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { + unsafe { ffi::roaring_bitmap_size_in_bytes(&bitmap.bitmap) } + } + + pub fn try_deserialize(buffer: &[u8]) -> Option { + unsafe { + let bitmap = ffi::roaring_bitmap_deserialize_safe( + buffer.as_ptr() as *const c_void, + buffer.len(), + ); + + if !bitmap.is_null() { + Some(Bitmap::take_heap(bitmap)) + } else { + None + } + } + } +} + +pub struct FrozenSerializer {} + +impl FrozenSerializer { + pub fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { + const REQUIRED_ALIGNMENT: usize = 32; + let len = Self::get_serialized_size_in_bytes(bitmap); + + let offset = dst.len(); + // Need to be able to add up to 31 extra bytes to align to 32 bytes + dst.reserve(len.checked_add(REQUIRED_ALIGNMENT - 1).unwrap()); + + let extra_offset = match (dst.as_ptr() as usize) % REQUIRED_ALIGNMENT { + 0 => 0, + r => REQUIRED_ALIGNMENT - r, + }; + let offset = offset.checked_add(extra_offset).unwrap(); + let total_len = offset.checked_add(len).unwrap(); + debug_assert!(dst.capacity() >= total_len); + + // we must initialize up to offset + dst.resize(offset, 0); + + unsafe { + ffi::roaring_bitmap_frozen_serialize( + &bitmap.bitmap, + dst.as_mut_ptr().add(offset).cast::(), + ); + dst.set_len(total_len); + } + + &dst[offset..total_len] + } + + pub fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { + unsafe { ffi::roaring_bitmap_frozen_size_in_bytes(&bitmap.bitmap) } + } +} From aef54e6b52ddda681f5c528998b1f674bddf932f Mon Sep 17 00:00:00 2001 From: Andreas Garnaes Date: Tue, 13 Jun 2023 13:21:39 +0200 Subject: [PATCH 2/6] Use traits and generic parameters for serialization --- croaring/benches/benches.rs | 16 ++--- croaring/src/bitmap/imp.rs | 37 ++++-------- croaring/src/bitmap/mod.rs | 2 +- croaring/src/bitmap/serialization.rs | 86 ++++++++++++++++++++++----- croaring/src/bitmap/view.rs | 51 ++-------------- croaring/src/lib.rs | 1 + croaring/src/treemap/serialization.rs | 18 +++--- croaring/tests/lib.rs | 18 +++--- 8 files changed, 114 insertions(+), 115 deletions(-) diff --git a/croaring/benches/benches.rs b/croaring/benches/benches.rs index f9a9e99..d7b233f 100644 --- a/croaring/benches/benches.rs +++ b/croaring/benches/benches.rs @@ -2,7 +2,7 @@ extern crate test; -use croaring::Bitmap; +use croaring::{Bitmap, Portable}; use test::Bencher; #[bench] @@ -314,7 +314,7 @@ fn bench_get_serialized_size_in_bytes(b: &mut Bencher) { bitmap.add(3); b.iter(|| { - bitmap.get_serialized_size_in_bytes(); + bitmap.get_serialized_size_in_bytes::(); }); } @@ -348,7 +348,7 @@ fn bench_serialize_100000(b: &mut Bencher) { let bitmap: Bitmap = (1..100000).collect(); b.iter(|| { - bitmap.serialize(); + bitmap.serialize::(); }); } @@ -357,26 +357,26 @@ fn bench_serialize_1000000(b: &mut Bencher) { let bitmap: Bitmap = (1..1000000).collect(); b.iter(|| { - bitmap.serialize(); + bitmap.serialize::(); }); } #[bench] fn bench_deserialize_100000(b: &mut Bencher) { let bitmap: Bitmap = (1..100000).collect(); - let serialized_buffer = bitmap.serialize(); + let serialized_buffer = bitmap.serialize::(); b.iter(|| { - Bitmap::deserialize(&serialized_buffer); + Bitmap::deserialize::(&serialized_buffer); }); } #[bench] fn bench_deserialize_1000000(b: &mut Bencher) { let bitmap: Bitmap = (1..1000000).collect(); - let serialized_buffer = bitmap.serialize(); + let serialized_buffer = bitmap.serialize::(); b.iter(|| { - Bitmap::deserialize(&serialized_buffer); + Bitmap::deserialize::(&serialized_buffer); }); } diff --git a/croaring/src/bitmap/imp.rs b/croaring/src/bitmap/imp.rs index 0552164..f853529 100644 --- a/croaring/src/bitmap/imp.rs +++ b/croaring/src/bitmap/imp.rs @@ -4,6 +4,7 @@ use std::convert::TryInto; use std::mem; use std::ops::{Bound, RangeBounds}; +use super::serialization::{Deserializer, Serializer}; use super::{Bitmap, Statistics}; impl Bitmap { @@ -736,15 +737,8 @@ impl Bitmap { /// Computes the serialized size in bytes of the Bitmap. #[inline] #[doc(alias = "roaring_bitmap_portable_size_in_bytes")] - pub fn get_serialized_size_in_bytes(&self) -> usize { - super::PortableSerializer::get_serialized_size_in_bytes(&self) - } - - /// Computes the serialized size in bytes of the Bitmap for the frozen format. - #[inline] - #[doc(alias = "roaring_bitmap_frozen_size_in_bytes")] - pub fn get_frozen_serialized_size_in_bytes(&self) -> usize { - super::FrozenSerializer::get_serialized_size_in_bytes(&self) + pub fn get_serialized_size_in_bytes(&self) -> usize { + S::get_serialized_size_in_bytes(&self) } /// Serializes a bitmap to a slice of bytes. @@ -764,9 +758,9 @@ impl Bitmap { /// ``` #[inline] #[doc(alias = "roaring_bitmap_portable_serialize")] - pub fn serialize(&self) -> Vec { + pub fn serialize(&self) -> Vec { let mut dst = Vec::new(); - self.serialize_into(&mut dst); + self.serialize_into::(&mut dst); dst } @@ -792,17 +786,8 @@ impl Bitmap { /// ``` #[inline] #[doc(alias = "roaring_bitmap_portable_serialize")] - pub fn serialize_into<'a>(&self, dst: &'a mut Vec) -> &'a [u8] { - super::PortableSerializer::serialize_into(self, dst) - } - - /// Serialize into the "frozen" format - /// - /// This has an odd API because it always returns a slice which is aligned to 32 bytes: - /// This means the returned slice may not start exactly at the beginning of the passed Vec - #[doc(alias = "roaring_bitmap_frozen_serialize")] - pub fn serialize_frozen_into<'a>(&self, dst: &'a mut Vec) -> &'a [u8] { - super::FrozenSerializer::serialize_into(self, dst) + pub fn serialize_into<'a, S: Serializer>(&self, dst: &'a mut Vec) -> &'a [u8] { + S::serialize_into(self, dst) } /// Given a serialized bitmap as slice of bytes returns a bitmap instance. @@ -827,8 +812,8 @@ impl Bitmap { /// ``` #[inline] #[doc(alias = "roaring_bitmap_portable_deserialize_safe")] - pub fn try_deserialize(buffer: &[u8]) -> Option { - super::PortableSerializer::try_deserialize(buffer) + pub fn try_deserialize(buffer: &[u8]) -> Option { + D::try_deserialize(buffer) } /// Given a serialized bitmap as slice of bytes returns a bitmap instance. @@ -836,8 +821,8 @@ impl Bitmap { /// /// On invalid input returns empty bitmap. #[inline] - pub fn deserialize(buffer: &[u8]) -> Self { - Self::try_deserialize(buffer).unwrap_or_else(Bitmap::create) + pub fn deserialize(buffer: &[u8]) -> Self { + Self::try_deserialize::(buffer).unwrap_or_else(Bitmap::create) } /// Creates a new bitmap from a slice of u32 integers diff --git a/croaring/src/bitmap/mod.rs b/croaring/src/bitmap/mod.rs index 97541a2..472b874 100644 --- a/croaring/src/bitmap/mod.rs +++ b/croaring/src/bitmap/mod.rs @@ -91,4 +91,4 @@ mod view; pub use self::iter::BitmapIterator; pub use self::lazy::LazyBitmap; -pub use self::serialization::{FrozenSerializer, NativeSerializer, PortableSerializer}; +pub use self::serialization::{Frozen, Native, Portable}; diff --git a/croaring/src/bitmap/serialization.rs b/croaring/src/bitmap/serialization.rs index 8719eed..14e038a 100644 --- a/croaring/src/bitmap/serialization.rs +++ b/croaring/src/bitmap/serialization.rs @@ -1,11 +1,24 @@ -use super::Bitmap; +use super::{Bitmap, BitmapView}; use std::ffi::{c_char, c_void}; -pub struct PortableSerializer {} +pub trait Serializer { + fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8]; + fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize; +} + +pub trait Deserializer { + fn try_deserialize(buffer: &[u8]) -> Option; +} + +pub trait ViewDeserializer { + unsafe fn deserialize_view(data: &[u8]) -> BitmapView<'_>; +} + +pub enum Portable {} -impl PortableSerializer { - pub fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { +impl Serializer for Portable { + fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { let len = Self::get_serialized_size_in_bytes(bitmap); dst.reserve(len); @@ -22,11 +35,13 @@ impl PortableSerializer { dst } - pub fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { + fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { unsafe { ffi::roaring_bitmap_portable_size_in_bytes(&bitmap.bitmap) } } +} - pub fn try_deserialize(buffer: &[u8]) -> Option { +impl Deserializer for Portable { + fn try_deserialize(buffer: &[u8]) -> Option { unsafe { let bitmap = ffi::roaring_bitmap_portable_deserialize_safe( buffer.as_ptr() as *const c_char, @@ -42,10 +57,32 @@ impl PortableSerializer { } } -pub struct NativeSerializer {} +impl ViewDeserializer for Portable { + /// Read bitmap from a serialized buffer + /// + /// This is meant to be compatible with the Java and Go versions + /// + /// # Safety + /// * `data` must be the result of serializing a roaring bitmap in portable mode + /// (following `https://github.com/RoaringBitmap/RoaringFormatSpec`), for example, with + /// [`Bitmap::serialize`] + /// * Using this function (or the returned bitmap in any way) may execute unaligned memory accesses + /// + unsafe fn deserialize_view<'a>(data: &'a [u8]) -> BitmapView { + // portable_deserialize_size does some amount of checks, and returns zero if data cannot be valid + debug_assert_ne!( + ffi::roaring_bitmap_portable_deserialize_size(data.as_ptr().cast(), data.len()), + 0, + ); + let roaring = ffi::roaring_bitmap_portable_deserialize_frozen(data.as_ptr().cast()); + BitmapView::take_heap(roaring) + } +} + +pub enum Native {} -impl NativeSerializer { - pub fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { +impl Serializer for Native { + fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { let len = Self::get_serialized_size_in_bytes(bitmap); dst.reserve(len); @@ -62,11 +99,13 @@ impl NativeSerializer { dst } - pub fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { + fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { unsafe { ffi::roaring_bitmap_size_in_bytes(&bitmap.bitmap) } } +} - pub fn try_deserialize(buffer: &[u8]) -> Option { +impl Deserializer for Native { + fn try_deserialize(buffer: &[u8]) -> Option { unsafe { let bitmap = ffi::roaring_bitmap_deserialize_safe( buffer.as_ptr() as *const c_void, @@ -82,10 +121,10 @@ impl NativeSerializer { } } -pub struct FrozenSerializer {} +pub enum Frozen {} -impl FrozenSerializer { - pub fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { +impl Serializer for Frozen { + fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { const REQUIRED_ALIGNMENT: usize = 32; let len = Self::get_serialized_size_in_bytes(bitmap); @@ -115,7 +154,24 @@ impl FrozenSerializer { &dst[offset..total_len] } - pub fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { + fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { unsafe { ffi::roaring_bitmap_frozen_size_in_bytes(&bitmap.bitmap) } } } + +impl ViewDeserializer for Frozen { + /// Create a frozen bitmap view using the passed data + /// + /// # Safety + /// * `data` must be the result of serializing a roaring bitmap in frozen mode + /// (in c with `roaring_bitmap_frozen_serialize`, or via [`Bitmap::serialize_frozen_into`]). + /// * Its beginning must be aligned by 32 bytes. + /// * data.len() must be equal exactly to the size of the frozen bitmap. + unsafe fn deserialize_view<'a>(data: &'a [u8]) -> BitmapView { + const REQUIRED_ALIGNMENT: usize = 32; + assert_eq!(data.as_ptr() as usize % REQUIRED_ALIGNMENT, 0); + + let roaring = ffi::roaring_bitmap_frozen_view(data.as_ptr().cast(), data.len()); + BitmapView::take_heap(roaring) + } +} diff --git a/croaring/src/bitmap/view.rs b/croaring/src/bitmap/view.rs index 29cb03a..00055bf 100644 --- a/croaring/src/bitmap/view.rs +++ b/croaring/src/bitmap/view.rs @@ -1,3 +1,4 @@ +use super::serialization::ViewDeserializer; use super::{Bitmap, BitmapView}; use ffi::roaring_bitmap_t; use std::marker::PhantomData; @@ -19,7 +20,7 @@ const fn original_bitmap_ptr(bitmap: &roaring_bitmap_t) -> *const roaring_bitmap impl<'a> BitmapView<'a> { #[inline] #[allow(clippy::assertions_on_constants)] - unsafe fn take_heap(p: *const roaring_bitmap_t) -> Self { + pub(crate) unsafe fn take_heap(p: *const roaring_bitmap_t) -> Self { // This depends somewhat heavily on the implementation of croaring, // In particular, that `roaring_bitmap_t` doesn't store any pointers into itself // (it can be moved safely), and a "frozen" bitmap is stored in an arena, and the @@ -44,44 +45,6 @@ impl<'a> BitmapView<'a> { } } - /// Create a frozen bitmap view using the passed data - /// - /// # Safety - /// * `data` must be the result of serializing a roaring bitmap in frozen mode - /// (in c with `roaring_bitmap_frozen_serialize`, or via [`Bitmap::serialize_frozen_into`]). - /// * Its beginning must be aligned by 32 bytes. - /// * data.len() must be equal exactly to the size of the frozen bitmap. - /// - /// # Examples - /// - /// ``` - /// use croaring::{Bitmap, BitmapView}; - /// let orig_bitmap = Bitmap::of(&[1, 2, 3, 4]); - /// let mut buf = Vec::new(); - /// let data: &[u8] = orig_bitmap.serialize_frozen_into(&mut buf); - /// let view = unsafe { BitmapView::deserialize_frozen(&data) }; - /// assert!(view.contains_range(1..=4)); - /// assert_eq!(orig_bitmap, view); - /// ``` - #[doc(alias = "roaring_bitmap_frozen_view")] - pub unsafe fn deserialize_frozen(data: &'a [u8]) -> Self { - const REQUIRED_ALIGNMENT: usize = 32; - assert_eq!(data.as_ptr() as usize % REQUIRED_ALIGNMENT, 0); - - let roaring = ffi::roaring_bitmap_frozen_view(data.as_ptr().cast(), data.len()); - Self::take_heap(roaring) - } - - /// Read bitmap from a serialized buffer - /// - /// This is meant to be compatible with the Java and Go versions - /// - /// # Safety - /// * `data` must be the result of serializing a roaring bitmap in portable mode - /// (following `https://github.com/RoaringBitmap/RoaringFormatSpec`), for example, with - /// [`Bitmap::serialize`] - /// * Using this function (or the returned bitmap in any way) may execute unaligned memory accesses - /// /// # Examples /// /// ``` @@ -93,14 +56,8 @@ impl<'a> BitmapView<'a> { /// assert_eq!(orig_bitmap, view); /// ``` #[doc(alias = "roaring_bitmap_portable_deserialize_frozen")] - pub unsafe fn deserialize(data: &'a [u8]) -> Self { - // portable_deserialize_size does some amount of checks, and returns zero if data cannot be valid - debug_assert_ne!( - ffi::roaring_bitmap_portable_deserialize_size(data.as_ptr().cast(), data.len()), - 0, - ); - let roaring = ffi::roaring_bitmap_portable_deserialize_frozen(data.as_ptr().cast()); - Self::take_heap(roaring) + pub unsafe fn deserialize(data: &'a [u8]) -> Self { + S::deserialize_view(data) } /// Create an owned, mutable bitmap from this view diff --git a/croaring/src/lib.rs b/croaring/src/lib.rs index 3753d43..35ad8be 100644 --- a/croaring/src/lib.rs +++ b/croaring/src/lib.rs @@ -8,3 +8,4 @@ pub use bitset::Bitset; pub use treemap::Treemap; pub use bitmap::BitmapView; +pub use bitmap::{Frozen, Native, Portable}; diff --git a/croaring/src/treemap/serialization.rs b/croaring/src/treemap/serialization.rs index 1d71836..d9d787d 100644 --- a/croaring/src/treemap/serialization.rs +++ b/croaring/src/treemap/serialization.rs @@ -1,5 +1,5 @@ -use crate::Bitmap; use crate::Treemap; +use crate::{Bitmap, Portable}; use byteorder::{BigEndian, NativeEndian, ReadBytesExt, WriteBytesExt}; use std::io::{Cursor, Result, Seek, SeekFrom}; @@ -28,7 +28,7 @@ impl NativeSerializer for Treemap { for (index, bitmap) in &self.map { buffer.write_u32::(*index)?; - let bitmap_buffer = bitmap.serialize(); + let bitmap_buffer = bitmap.serialize::(); buffer.extend(bitmap_buffer); } @@ -42,9 +42,9 @@ impl NativeSerializer for Treemap { for _ in 0..bitmap_count { let index = cursor.read_u32::()?; - let bitmap = Bitmap::deserialize(&buffer[cursor.position() as usize..]); + let bitmap = Bitmap::deserialize::(&buffer[cursor.position() as usize..]); cursor.seek(SeekFrom::Current( - bitmap.get_serialized_size_in_bytes() as i64 + bitmap.get_serialized_size_in_bytes::() as i64, ))?; treemap.map.insert(index, bitmap); } @@ -75,7 +75,7 @@ impl NativeSerializer for Treemap { fn get_serialized_size_in_bytes(&self) -> usize { self.map.iter().fold( size_of::() + self.map.len() * size_of::(), - |sum, (_, bitmap)| sum + bitmap.get_serialized_size_in_bytes(), + |sum, (_, bitmap)| sum + bitmap.get_serialized_size_in_bytes::(), ) } } @@ -101,7 +101,7 @@ impl JvmSerializer for Treemap { for (index, bitmap) in &self.map { buffer.write_u32::(*index)?; - let bitmap_buffer = bitmap.serialize(); + let bitmap_buffer = bitmap.serialize::(); buffer.extend(bitmap_buffer); } @@ -117,9 +117,9 @@ impl JvmSerializer for Treemap { for _ in 0..bitmap_count { let index = cursor.read_u32::()?; - let bitmap = Bitmap::deserialize(&buffer[cursor.position() as usize..]); + let bitmap = Bitmap::deserialize::(&buffer[cursor.position() as usize..]); cursor.seek(SeekFrom::Current( - bitmap.get_serialized_size_in_bytes() as i64 + bitmap.get_serialized_size_in_bytes::() as i64, ))?; treemap.map.insert(index, bitmap); } @@ -150,7 +150,7 @@ impl JvmSerializer for Treemap { fn get_serialized_size_in_bytes(&self) -> usize { self.map.iter().fold( size_of::() + size_of::() + self.map.len() * size_of::(), - |sum, (_, bitmap)| sum + bitmap.get_serialized_size_in_bytes(), + |sum, (_, bitmap)| sum + bitmap.get_serialized_size_in_bytes::(), ) } } diff --git a/croaring/tests/lib.rs b/croaring/tests/lib.rs index 994961b..240fd68 100644 --- a/croaring/tests/lib.rs +++ b/croaring/tests/lib.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use std::{fs, iter, u32}; -use croaring::{Bitmap, BitmapView, Treemap}; +use croaring::{Bitmap, BitmapView, Frozen, Portable, Treemap}; use proptest::prelude::*; // borrowed and adapted from https://github.com/Nemo157/roaring-rs/blob/5089f180ca7e17db25f5c58023f4460d973e747f/tests/lib.rs#L7-L37 @@ -105,7 +105,7 @@ fn expected_serialized_bitmap() -> Bitmap { #[test] fn test_portable_view() { let buffer = fs::read("tests/data/portable_bitmap.bin").unwrap(); - let bitmap = unsafe { BitmapView::deserialize(&buffer) }; + let bitmap = unsafe { BitmapView::deserialize::(&buffer) }; let expected = expected_serialized_bitmap(); assert_eq!(bitmap, expected); assert!(bitmap.iter().eq(expected.iter())) @@ -119,7 +119,7 @@ fn test_frozen_view() { let offset = 32 - (buffer.as_ptr() as usize) % 32; buffer.splice(..0, iter::repeat(0).take(offset)); - let bitmap = unsafe { BitmapView::deserialize_frozen(&buffer[offset..]) }; + let bitmap = unsafe { BitmapView::deserialize::(&buffer[offset..]) }; let expected = expected_serialized_bitmap(); assert_eq!(bitmap, expected); assert!(bitmap.iter().eq(expected.iter())) @@ -230,9 +230,9 @@ proptest! { ) { let original = Bitmap::of(&indices); - let buffer = original.serialize(); + let buffer = original.serialize::(); - let deserialized = Bitmap::deserialize(&buffer); + let deserialized = Bitmap::deserialize::(&buffer); prop_assert_eq!(original , deserialized); } @@ -276,8 +276,8 @@ proptest! { use croaring::BitmapView; let original = Bitmap::of(&indices); - let serialized = original.serialize(); - let deserialized = unsafe { BitmapView::deserialize(&serialized) }; + let serialized = original.serialize::(); + let deserialized = unsafe { BitmapView::deserialize::(&serialized) }; assert_eq!(&original, &*deserialized); assert!(original.iter().eq(deserialized.iter())); } @@ -290,8 +290,8 @@ proptest! { let original = Bitmap::of(&indices); let mut buf = Vec::new(); - let serialized: &[u8] = original.serialize_frozen_into(&mut buf); - let deserialized = unsafe { BitmapView::deserialize_frozen(serialized) }; + let serialized: &[u8] = original.serialize_into::(&mut buf); + let deserialized = unsafe { BitmapView::deserialize::(serialized) }; assert_eq!(&original, &*deserialized); assert!(original.iter().eq(deserialized.iter())); } From b65070d9c52e44c8d48acc3e75f19799e119fcad Mon Sep 17 00:00:00 2001 From: Andreas Garnaes Date: Tue, 20 Jun 2023 12:49:34 +0200 Subject: [PATCH 3/6] Fix docs --- croaring/src/bitmap/imp.rs | 37 +++++++++++++-------------- croaring/src/bitmap/serialization.rs | 38 ++++++++++++++++++++++++++++ croaring/src/bitmap/view.rs | 13 +++++----- 3 files changed, 61 insertions(+), 27 deletions(-) diff --git a/croaring/src/bitmap/imp.rs b/croaring/src/bitmap/imp.rs index f853529..d33e81b 100644 --- a/croaring/src/bitmap/imp.rs +++ b/croaring/src/bitmap/imp.rs @@ -734,37 +734,35 @@ impl Bitmap { buffer } - /// Computes the serialized size in bytes of the Bitmap. + /// Computes the serialized size in bytes of the Bitmap in format `S`. #[inline] - #[doc(alias = "roaring_bitmap_portable_size_in_bytes")] pub fn get_serialized_size_in_bytes(&self) -> usize { S::get_serialized_size_in_bytes(&self) } - /// Serializes a bitmap to a slice of bytes. + /// Serializes a bitmap to a slice of bytes in format `S`. /// /// # Examples /// /// ``` - /// use croaring::Bitmap; + /// use croaring::{Bitmap, Portable}; /// /// let original_bitmap: Bitmap = (1..5).collect(); /// - /// let serialized_buffer = original_bitmap.serialize(); + /// let serialized_buffer = original_bitmap.serialize::(); /// - /// let deserialized_bitmap = Bitmap::deserialize(&serialized_buffer); + /// let deserialized_bitmap = Bitmap::deserialize::(&serialized_buffer); /// /// assert_eq!(original_bitmap, deserialized_bitmap); /// ``` #[inline] - #[doc(alias = "roaring_bitmap_portable_serialize")] pub fn serialize(&self) -> Vec { let mut dst = Vec::new(); self.serialize_into::(&mut dst); dst } - /// Serializes a bitmap to a slice of bytes, re-using existing capacity + /// Serializes a bitmap to a slice of bytes in format `S`, re-using existing capacity /// /// `dst` is not cleared, data is added after any existing data. Returns the added slice of `dst`. /// If `dst` is empty, it is guaranteed to hold only the serialized data after this call @@ -772,7 +770,7 @@ impl Bitmap { /// # Examples /// /// ``` - /// use croaring::Bitmap; + /// use croaring::{Bitmap, Portable}; /// /// let original_bitmap_1: Bitmap = (1..5).collect(); /// let original_bitmap_2: Bitmap = (1..10).collect(); @@ -780,7 +778,7 @@ impl Bitmap { /// let mut data = Vec::new(); /// for bitmap in [original_bitmap_1, original_bitmap_2] { /// data.clear(); - /// bitmap.serialize_into(&mut data); + /// bitmap.serialize_into::(&mut data); /// // do something with data /// } /// ``` @@ -790,7 +788,7 @@ impl Bitmap { S::serialize_into(self, dst) } - /// Given a serialized bitmap as slice of bytes returns a bitmap instance. + /// Given a serialized bitmap as slice of bytes in format `S`, returns a `Bitmap` instance. /// See example of [`Self::serialize`] function. /// /// On invalid input returns None. @@ -798,25 +796,24 @@ impl Bitmap { /// # Examples /// /// ``` - /// use croaring::Bitmap; + /// use croaring::{Bitmap, Portable}; /// /// let original_bitmap: Bitmap = (1..5).collect(); - /// let serialized_buffer = original_bitmap.serialize(); + /// let serialized_buffer = original_bitmap.serialize::(); /// - /// let deserialized_bitmap = Bitmap::try_deserialize(&serialized_buffer); + /// let deserialized_bitmap = Bitmap::try_deserialize::(&serialized_buffer); /// assert_eq!(original_bitmap, deserialized_bitmap.unwrap()); /// /// let invalid_buffer: Vec = vec![3]; - /// let deserialized_bitmap = Bitmap::try_deserialize(&invalid_buffer); + /// let deserialized_bitmap = Bitmap::try_deserialize::(&invalid_buffer); /// assert!(deserialized_bitmap.is_none()); /// ``` #[inline] - #[doc(alias = "roaring_bitmap_portable_deserialize_safe")] pub fn try_deserialize(buffer: &[u8]) -> Option { D::try_deserialize(buffer) } - /// Given a serialized bitmap as slice of bytes returns a bitmap instance. + /// Given a serialized bitmap as slice of bytes in format `S `, returns a bitmap instance. /// See example of [`Self::serialize`] function. /// /// On invalid input returns empty bitmap. @@ -963,14 +960,14 @@ impl Bitmap { /// # Examples /// /// ``` - /// use croaring::Bitmap; + /// use croaring::{Bitmap, Portable}; /// /// let mut bitmap: Bitmap = (100..1000).collect(); /// /// assert_eq!(bitmap.cardinality(), 900); - /// let old_size = bitmap.get_serialized_size_in_bytes(); + /// let old_size = bitmap.get_serialized_size_in_bytes::(); /// assert!(bitmap.run_optimize()); - /// let new_size = bitmap.get_serialized_size_in_bytes(); + /// let new_size = bitmap.get_serialized_size_in_bytes::(); /// assert!(new_size < old_size); /// ``` #[inline] diff --git a/croaring/src/bitmap/serialization.rs b/croaring/src/bitmap/serialization.rs index 14e038a..93aa11c 100644 --- a/croaring/src/bitmap/serialization.rs +++ b/croaring/src/bitmap/serialization.rs @@ -15,9 +15,14 @@ pub trait ViewDeserializer { unsafe fn deserialize_view(data: &[u8]) -> BitmapView<'_>; } +/// The `Portable` format is meant to be compatible with other roaring bitmap libraries, such as Go or Java. +/// It's defined here: https://github.com/RoaringBitmap/RoaringFormatSpec pub enum Portable {} impl Serializer for Portable { + /// Serializes a bitmap to a slice of bytes in portable format. + /// See [`Bitmap::serialize_into`] for examples. + #[doc(alias = "roaring_bitmap_portable_serialize")] fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { let len = Self::get_serialized_size_in_bytes(bitmap); @@ -35,12 +40,18 @@ impl Serializer for Portable { dst } + /// Computes the serialized size in bytes of the Bitmap in portable format. + /// See [`Bitmap::get_size_in_bytes`] for examples. + #[doc(alias = "roaring_bitmap_portable_size_in_bytes")] fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { unsafe { ffi::roaring_bitmap_portable_size_in_bytes(&bitmap.bitmap) } } } impl Deserializer for Portable { + /// Given a serialized bitmap as slice of bytes in portable format, returns a `Bitmap` instance. + /// See [`Bitmap::try_deserialize`] for examples. + #[doc(alias = "roaring_bitmap_portable_deserialize_safe")] fn try_deserialize(buffer: &[u8]) -> Option { unsafe { let bitmap = ffi::roaring_bitmap_portable_deserialize_safe( @@ -68,6 +79,7 @@ impl ViewDeserializer for Portable { /// [`Bitmap::serialize`] /// * Using this function (or the returned bitmap in any way) may execute unaligned memory accesses /// + #[doc(alias = "roaring_bitmap_portable_deserialize_frozen")] unsafe fn deserialize_view<'a>(data: &'a [u8]) -> BitmapView { // portable_deserialize_size does some amount of checks, and returns zero if data cannot be valid debug_assert_ne!( @@ -79,9 +91,15 @@ impl ViewDeserializer for Portable { } } +/// The `Native` format format can sometimes be more space efficient than [`Portable`], e.g. when +/// the data is sparse. It's not compatible with Java and Go implementations. Use [`Portable`] for +/// that purpose. pub enum Native {} impl Serializer for Native { + /// Serializes a bitmap to a slice of bytes in native format. + /// See [`Bitmap::serialize_into`] for examples. + #[doc(alias = "roaring_bitmap_serialize")] fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { let len = Self::get_serialized_size_in_bytes(bitmap); @@ -99,12 +117,18 @@ impl Serializer for Native { dst } + /// Computes the serialized size in bytes of the Bitmap in native format. + /// See [`Bitmap::get_size_in_bytes`] for examples. + #[doc(alias = "roaring_bitmap_size_in_bytes")] fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { unsafe { ffi::roaring_bitmap_size_in_bytes(&bitmap.bitmap) } } } impl Deserializer for Native { + /// Given a serialized bitmap as slice of bytes in native format, returns a `Bitmap` instance. + /// See [`Bitmap::try_deserialize`] for examples. + #[doc(alias = "roaring_bitmap_deserialize_safe")] fn try_deserialize(buffer: &[u8]) -> Option { unsafe { let bitmap = ffi::roaring_bitmap_deserialize_safe( @@ -121,9 +145,18 @@ impl Deserializer for Native { } } +/// The `Frozen` format imitates memory layout of the underlying C library. +/// This reduces amount of allocations and copying required during deserialization, though +/// `Portable` offers comparable performance. pub enum Frozen {} impl Serializer for Frozen { + /// Serializes a bitmap to a slice of bytes in "frozen" format. + /// + /// This has an odd API because it always returns a slice which is aligned to 32 bytes: + /// This means the returned slice may not start exactly at the beginning of the passed `Vec` + /// See [`Bitmap::serialize_into`] for examples. + #[doc(alias = "roaring_bitmap_frozen_serialize")] fn serialize_into<'a>(bitmap: &Bitmap, dst: &'a mut Vec) -> &'a [u8] { const REQUIRED_ALIGNMENT: usize = 32; let len = Self::get_serialized_size_in_bytes(bitmap); @@ -154,6 +187,9 @@ impl Serializer for Frozen { &dst[offset..total_len] } + /// Computes the serialized size in bytes of the Bitmap in frozen format. + /// See [`Bitmap::get_size_in_bytes`] for examples. + #[doc(alias = "roaring_bitmap_frozen_size_in_bytes")] fn get_serialized_size_in_bytes(bitmap: &Bitmap) -> usize { unsafe { ffi::roaring_bitmap_frozen_size_in_bytes(&bitmap.bitmap) } } @@ -167,6 +203,8 @@ impl ViewDeserializer for Frozen { /// (in c with `roaring_bitmap_frozen_serialize`, or via [`Bitmap::serialize_frozen_into`]). /// * Its beginning must be aligned by 32 bytes. /// * data.len() must be equal exactly to the size of the frozen bitmap. + /// + /// See [`BitmapView::deserialize`] for examples. unsafe fn deserialize_view<'a>(data: &'a [u8]) -> BitmapView { const REQUIRED_ALIGNMENT: usize = 32; assert_eq!(data.as_ptr() as usize % REQUIRED_ALIGNMENT, 0); diff --git a/croaring/src/bitmap/view.rs b/croaring/src/bitmap/view.rs index 00055bf..74b9b7d 100644 --- a/croaring/src/bitmap/view.rs +++ b/croaring/src/bitmap/view.rs @@ -48,14 +48,13 @@ impl<'a> BitmapView<'a> { /// # Examples /// /// ``` - /// use croaring::{Bitmap, BitmapView}; + /// use croaring::{Bitmap, BitmapView, Portable}; /// let orig_bitmap = Bitmap::of(&[1, 2, 3, 4]); - /// let data: Vec = orig_bitmap.serialize(); - /// let view = unsafe { BitmapView::deserialize(&data) }; + /// let data: Vec = orig_bitmap.serialize::(); + /// let view = unsafe { BitmapView::deserialize::(&data) }; /// assert!(view.contains_range(1..=4)); /// assert_eq!(orig_bitmap, view); /// ``` - #[doc(alias = "roaring_bitmap_portable_deserialize_frozen")] pub unsafe fn deserialize(data: &'a [u8]) -> Self { S::deserialize_view(data) } @@ -65,11 +64,11 @@ impl<'a> BitmapView<'a> { /// # Examples /// /// ``` - /// use croaring::{Bitmap, BitmapView}; + /// use croaring::{Bitmap, BitmapView, Portable}; /// /// let orig_bitmap = Bitmap::of(&[1, 2, 3, 4]); - /// let data = orig_bitmap.serialize(); - /// let view: BitmapView = unsafe { BitmapView::deserialize(&data) }; + /// let data = orig_bitmap.serialize::(); + /// let view: BitmapView = unsafe { BitmapView::deserialize::(&data) }; /// # assert_eq!(view, orig_bitmap); /// let mut mutable_bitmap: Bitmap = view.to_bitmap(); /// assert_eq!(view, mutable_bitmap); From 6e927bcaf4556db42f34e57b140762f2012f0157 Mon Sep 17 00:00:00 2001 From: Andreas Garnaes Date: Sat, 24 Jun 2023 17:01:47 +0200 Subject: [PATCH 4/6] Fix and add tests for native serialization --- croaring/tests/data/create_serialization.c | 9 ++++++++ croaring/tests/data/native_bitmap.bin | Bin 0 -> 8224 bytes croaring/tests/lib.rs | 24 ++++++++++++++++++++- fuzz/fuzz_targets/against_bitvec.rs | 9 +++++--- fuzz/fuzz_targets/arbitrary_ops/mod.rs | 3 ++- fuzz/fuzz_targets/fuzz_ops.rs | 19 +++++++++------- 6 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 croaring/tests/data/native_bitmap.bin diff --git a/croaring/tests/data/create_serialization.c b/croaring/tests/data/create_serialization.c index 8dd8d14..5732760 100644 --- a/croaring/tests/data/create_serialization.c +++ b/croaring/tests/data/create_serialization.c @@ -26,6 +26,14 @@ void write_portable(const roaring_bitmap_t *b) { roaring_free(data); } +void write_native(const roaring_bitmap_t *b) { + size_t size = roaring_bitmap_size_in_bytes(b); + char *data = roaring_malloc(size); + roaring_bitmap_serialize(b, data); + write_file("native_bitmap.bin", data, size); + roaring_free(data); +} + int main(void) { int i; @@ -45,6 +53,7 @@ int main(void) { write_frozen(b); write_portable(b); + write_native(b); roaring_bitmap_free(b); } \ No newline at end of file diff --git a/croaring/tests/data/native_bitmap.bin b/croaring/tests/data/native_bitmap.bin new file mode 100644 index 0000000000000000000000000000000000000000..99d96e7defd7c2d47633164c84efc1f17ac1267a GIT binary patch literal 8224 zcmeIuF%5uF5JbV{5)$OlqvO{|K>`+Vh=c=VkjyqO+fHq3WlH+~vZRm7`_5dwCv%1o oAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0>2k{03%hX5dZ)H literal 0 HcmV?d00001 diff --git a/croaring/tests/lib.rs b/croaring/tests/lib.rs index 240fd68..ef25e7e 100644 --- a/croaring/tests/lib.rs +++ b/croaring/tests/lib.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use std::{fs, iter, u32}; -use croaring::{Bitmap, BitmapView, Frozen, Portable, Treemap}; +use croaring::{Bitmap, BitmapView, Frozen, Native, Portable, Treemap}; use proptest::prelude::*; // borrowed and adapted from https://github.com/Nemo157/roaring-rs/blob/5089f180ca7e17db25f5c58023f4460d973e747f/tests/lib.rs#L7-L37 @@ -111,6 +111,15 @@ fn test_portable_view() { assert!(bitmap.iter().eq(expected.iter())) } +#[test] +fn test_native() { + let buffer = fs::read("tests/data/native_bitmap.bin").unwrap(); + let bitmap = Bitmap::deserialize::(&buffer); + let expected = expected_serialized_bitmap(); + assert_eq!(bitmap, expected); + assert!(bitmap.iter().eq(expected.iter())) +} + #[test] fn test_frozen_view() { let mut buffer = fs::read("tests/data/frozen_bitmap.bin").unwrap(); @@ -282,6 +291,19 @@ proptest! { assert!(original.iter().eq(deserialized.iter())); } + #[test] + fn native_bitmap_roundtrip( + indices in prop::collection::vec(proptest::num::u32::ANY, 0..3000) + ) { + use croaring::Bitmap; + + let original = Bitmap::of(&indices); + let serialized = original.serialize::(); + let deserialized = Bitmap::deserialize::(&serialized[..]); + assert_eq!(&original, &deserialized); + assert!(original.iter().eq(deserialized.iter())); + } + #[test] fn frozen_bitmap_roundtrip( indices in prop::collection::vec(proptest::num::u32::ANY, 0..3000) diff --git a/fuzz/fuzz_targets/against_bitvec.rs b/fuzz/fuzz_targets/against_bitvec.rs index 02e28be..c89d1d8 100644 --- a/fuzz/fuzz_targets/against_bitvec.rs +++ b/fuzz/fuzz_targets/against_bitvec.rs @@ -75,11 +75,14 @@ impl ReadBitmapOp { let vec_iter = b.to_vec(); assert!(vec_iter.into_iter().eq(v.iter_ones().map(|i| i as u32))); } - ReadBitmapOp::GetSerializedSizeInBytes => { - b.get_serialized_size_in_bytes(); + ReadBitmapOp::GetPortableSerializedSizeInBytes => { + b.get_serialized_size_in_bytes::(); + } + ReadBitmapOp::GetNativeSerializedSizeInBytes => { + b.get_serialized_size_in_bytes::(); } ReadBitmapOp::GetFrozenSerializedSizeInBytes => { - b.get_frozen_serialized_size_in_bytes(); + b.get_serialized_size_in_bytes::(); } ReadBitmapOp::IsEmpty => { assert_eq!(v.not_any(), b.is_empty()); diff --git a/fuzz/fuzz_targets/arbitrary_ops/mod.rs b/fuzz/fuzz_targets/arbitrary_ops/mod.rs index b028fb1..a522134 100644 --- a/fuzz/fuzz_targets/arbitrary_ops/mod.rs +++ b/fuzz/fuzz_targets/arbitrary_ops/mod.rs @@ -42,7 +42,8 @@ pub enum ReadBitmapOp { Cardinality, Flip(RangeInclusive), ToVec, - GetSerializedSizeInBytes, + GetPortableSerializedSizeInBytes, + GetNativeSerializedSizeInBytes, GetFrozenSerializedSizeInBytes, IsEmpty, AddOffset(i64), diff --git a/fuzz/fuzz_targets/fuzz_ops.rs b/fuzz/fuzz_targets/fuzz_ops.rs index a88cd7e..165d9ea 100644 --- a/fuzz/fuzz_targets/fuzz_ops.rs +++ b/fuzz/fuzz_targets/fuzz_ops.rs @@ -1,7 +1,7 @@ #![no_main] use crate::arbitrary_ops::*; -use croaring::{Bitmap, BitmapView}; +use croaring::{Bitmap, BitmapView, Frozen, Native, Portable}; use libfuzzer_sys::arbitrary; use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; @@ -37,11 +37,11 @@ fuzz_target!(|input: FuzzInput| { fn check_serialized(lhs: &mut Bitmap, to_compare: &Bitmap, v: &mut Vec, input: &FuzzInput) { v.clear(); - let data = to_compare.serialize(); - let data2 = to_compare.serialize_frozen_into(v); - let view1 = unsafe { BitmapView::deserialize(&data) }; + let data = to_compare.serialize::(); + let data2 = to_compare.serialize_into::(v); + let view1 = unsafe { BitmapView::deserialize::(&data) }; assert_eq!(view1, *to_compare); - let view2 = unsafe { BitmapView::deserialize_frozen(data2) }; + let view2 = unsafe { BitmapView::deserialize::(data2) }; assert_eq!(view2, *to_compare); for op in &input.view_ops { @@ -83,11 +83,14 @@ impl ReadBitmapOp { ReadBitmapOp::ToVec => { drop(b.to_vec()); } - ReadBitmapOp::GetSerializedSizeInBytes => { - b.get_serialized_size_in_bytes(); + ReadBitmapOp::GetPortableSerializedSizeInBytes => { + b.get_serialized_size_in_bytes::(); + } + ReadBitmapOp::GetNativeSerializedSizeInBytes => { + b.get_serialized_size_in_bytes::(); } ReadBitmapOp::GetFrozenSerializedSizeInBytes => { - b.get_frozen_serialized_size_in_bytes(); + b.get_serialized_size_in_bytes::(); } ReadBitmapOp::IsEmpty => { assert_eq!(b.is_empty(), b.cardinality() == 0); From a4559452e9148ce58f22cde918b82294bc87f929 Mon Sep 17 00:00:00 2001 From: Andreas Garnaes Date: Sun, 25 Jun 2023 18:36:18 +0200 Subject: [PATCH 5/6] Import formats in fuzz tests --- fuzz/fuzz_targets/against_bitvec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/fuzz_targets/against_bitvec.rs b/fuzz/fuzz_targets/against_bitvec.rs index c89d1d8..46b30dd 100644 --- a/fuzz/fuzz_targets/against_bitvec.rs +++ b/fuzz/fuzz_targets/against_bitvec.rs @@ -2,7 +2,7 @@ use crate::arbitrary_ops::*; use bitvec::prelude::*; -use croaring::Bitmap; +use croaring::{Bitmap, Native, Portable}; use libfuzzer_sys::arbitrary; use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; From 3138209c9b54b8e88770227c36d98c61ba5fb340 Mon Sep 17 00:00:00 2001 From: Andreas Garnaes Date: Sun, 25 Jun 2023 19:46:48 +0200 Subject: [PATCH 6/6] Fix imports for against_bitvec --- fuzz/fuzz_targets/against_bitvec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/fuzz_targets/against_bitvec.rs b/fuzz/fuzz_targets/against_bitvec.rs index 46b30dd..abc1608 100644 --- a/fuzz/fuzz_targets/against_bitvec.rs +++ b/fuzz/fuzz_targets/against_bitvec.rs @@ -2,7 +2,7 @@ use crate::arbitrary_ops::*; use bitvec::prelude::*; -use croaring::{Bitmap, Native, Portable}; +use croaring::{Bitmap, Frozen, Native, Portable}; use libfuzzer_sys::arbitrary; use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target;