From 49de417265a8b61c17a363e71cf6b2a9cd0e4614 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 16 Jan 2019 16:24:49 +0100 Subject: [PATCH] Implement CFixedVarIntsBitSet and CAutoBitSet --- src/serialize.h | 101 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/src/serialize.h b/src/serialize.h index f87009855a2ab..d2104672d8999 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -206,6 +206,8 @@ template inline void Unserialize(Stream& s, double& a ) { a = template inline void Serialize(Stream& s, bool a) { char f=a; ser_writedata8(s, f); } template inline void Unserialize(Stream& s, bool& a) { char f=ser_readdata8(s); a=f; } +template size_t GetSerializeSize(const T& t, int nType, int nVersion = 0); +template size_t GetSerializeSize(const S& s, const T& t); @@ -359,6 +361,8 @@ I ReadVarInt(Stream& is) #define FLATDATA(obj) REF(CFlatData((char*)&(obj), (char*)&(obj) + sizeof(obj))) #define FIXEDBITSET(obj, size) REF(CFixedBitSet(REF(obj), (size))) #define DYNBITSET(obj) REF(CDynamicBitSet(REF(obj))) +#define FIXEDVARINTSBITSET(obj, size) REF(CFixedVarIntsBitSet(REF(obj), (size))) +#define AUTOBITSET(obj, size) REF(CAutoBitSet(REF(obj), (size))) #define VARINT(obj) REF(WrapVarInt(REF(obj))) #define COMPACTSIZE(obj) REF(CCompactSize(REF(obj))) #define LIMITED_STRING(obj,n) REF(LimitedString< n >(REF(obj))) @@ -464,6 +468,101 @@ class CDynamicBitSet } }; +/** + * Stores a fixed size bitset as a series of VarInts. Each VarInt is an offset from the last entry and the sum of the + * last entry and the offset gives an index into the bitset for a set bit. The series of VarInts ends with a 0. + */ +class CFixedVarIntsBitSet +{ +protected: + std::vector& vec; + size_t size; + +public: + CFixedVarIntsBitSet(std::vector& vecIn, size_t sizeIn) : vec(vecIn), size(sizeIn) {} + + template + void Serialize(Stream& s) const + { + int32_t last = -1; + for (int32_t i = 0; i < (int32_t)vec.size(); i++) { + if (vec[i]) { + WriteVarInt(s, (uint32_t)(i - last)); + last = i; + } + } + WriteVarInt(s, 0); // stopper + } + + template + void Unserialize(Stream& s) + { + vec.assign(size, false); + + int32_t last = -1; + while(true) { + uint32_t offset = ReadVarInt(s); + if (offset == 0) { + break; + } + int32_t idx = last + offset; + if (idx >= size) { + throw std::ios_base::failure("out of bounds index"); + } + if (last != -1 && idx <= last) { + throw std::ios_base::failure("offset overflow"); + } + vec[idx] = true; + last = idx; + } + } +}; + +/** + * Serializes either as a CFixedBitSet or CFixedVarIntsBitSet, depending on which would give a smaller size + */ +class CAutoBitSet +{ +protected: + std::vector& vec; + size_t size; + +public: + explicit CAutoBitSet(std::vector& vecIn, size_t sizeIn) : vec(vecIn), size(sizeIn) {} + + template + void Serialize(Stream& s) const + { + assert(vec.size() == size); + + size_t size1 = ::GetSerializeSize(s, CFixedBitSet(vec, size)); + size_t size2 = ::GetSerializeSize(s, CFixedVarIntsBitSet(vec, size)); + + if (size1 < size2) { + ser_writedata8(s, 0); + s << FIXEDBITSET(vec, vec.size()); + } else { + ser_writedata8(s, 1); + s << FIXEDVARINTSBITSET(vec, vec.size()); + } + } + + template + void Unserialize(Stream& s) + { + uint8_t isVarInts = ser_readdata8(s); + if (isVarInts != 0 && isVarInts != 1) { + throw std::ios_base::failure("invalid value for isVarInts byte"); + } + + if (!isVarInts) { + s >> FIXEDBITSET(vec, size); + } else { + s >> FIXEDVARINTSBITSET(vec, size); + } + } +}; + template class CVarInt { @@ -1093,7 +1192,7 @@ inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize) } template -size_t GetSerializeSize(const T& t, int nType, int nVersion = 0) +size_t GetSerializeSize(const T& t, int nType, int nVersion) { return (CSizeComputer(nType, nVersion) << t).size(); }