Serialization improvements #10785

Open
wants to merge 26 commits into
from
Commits
Jump to file or symbol
Failed to load files and symbols.
+608 −763
Split
View
@@ -43,15 +43,7 @@ class CBanEntry
nCreateTime = nCreateTimeIn;
}
- ADD_SERIALIZE_METHODS;
-
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITE(this->nVersion);
- READWRITE(nCreateTime);
- READWRITE(nBanUntil);
- READWRITE(banReason);
- }
+ SERIALIZE_METHODS(obj) { READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, obj.banReason); }
void SetNull()
{
View
@@ -55,14 +55,10 @@ class CAddrInfo : public CAddress
public:
- ADD_SERIALIZE_METHODS;
-
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITE(*(CAddress*)this);
- READWRITE(source);
- READWRITE(nLastSuccess);
- READWRITE(nAttempts);
+ SERIALIZE_METHODS(obj)
+ {
+ READWRITEAS(obj, CAddress);
+ READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts);
}
void Init()
View
@@ -12,58 +12,70 @@
class CTxMemPool;
// Dumb helper to handle CTransaction compression at serialize-time
-struct TransactionCompressor {
+template<typename T>
+struct TransactionCompressWrapper {
private:
- CTransactionRef& tx;
+ T& tx;
public:
- TransactionCompressor(CTransactionRef& txIn) : tx(txIn) {}
-
- ADD_SERIALIZE_METHODS;
+ TransactionCompressWrapper(T& txIn) : tx(txIn) {}
+ SERIALIZE_METHODS(obj) { READWRITE(obj.tx); }
+};
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITE(tx); //TODO: Compress tx encoding
+template<typename I>
+struct Uint48Wrapper {
+private:
+ I& m_int;
+public:
+ Uint48Wrapper(I& i) : m_int(i) {}
+ template <typename Stream> void Serialize(Stream& s) const
+ {
+ uint32_t lsb = m_int & 0xffffffff;
+ uint16_t msb = (m_int >> 32) & 0xffff;
+ s << lsb << msb;
+ }
+ template <typename Stream> void Unserialize(Stream& s)
+ {
+ uint32_t lsb;
+ uint16_t msb;
+ s >> lsb >> msb;
+ m_int = (uint64_t(msb) << 32) | uint64_t(lsb);
}
};
+template<typename T>
+static inline TransactionCompressWrapper<T> TransactionCompressor(T& tx) { return TransactionCompressWrapper<T>(tx); }
+
class BlockTransactionsRequest {
public:
// A BlockTransactionsRequest message
uint256 blockhash;
std::vector<uint16_t> indexes;
- ADD_SERIALIZE_METHODS;
-
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITE(blockhash);
+ template <typename Stream>
+ void Serialize(Stream& s) const
+ {
+ s << blockhash;
uint64_t indexes_size = (uint64_t)indexes.size();
- READWRITE(COMPACTSIZE(indexes_size));
- if (ser_action.ForRead()) {
- size_t i = 0;
- while (indexes.size() < indexes_size) {
- indexes.resize(std::min((uint64_t)(1000 + indexes.size()), indexes_size));
- for (; i < indexes.size(); i++) {
- uint64_t index = 0;
- READWRITE(COMPACTSIZE(index));
- if (index > std::numeric_limits<uint16_t>::max())
- throw std::ios_base::failure("index overflowed 16 bits");
- indexes[i] = index;
- }
- }
-
- uint16_t offset = 0;
- for (size_t j = 0; j < indexes.size(); j++) {
- if (uint64_t(indexes[j]) + uint64_t(offset) > std::numeric_limits<uint16_t>::max())
- throw std::ios_base::failure("indexes overflowed 16 bits");
- indexes[j] = indexes[j] + offset;
- offset = indexes[j] + 1;
- }
- } else {
- for (size_t i = 0; i < indexes.size(); i++) {
- uint64_t index = indexes[i] - (i == 0 ? 0 : (indexes[i - 1] + 1));
- READWRITE(COMPACTSIZE(index));
- }
+ s << COMPACTSIZE(indexes_size);
+ for (size_t i = 0; i < indexes.size(); i++) {
+ const uint64_t index = indexes[i] - (i == 0 ? 0 : (indexes[i - 1] + 1));
+ s << COMPACTSIZE(index);
+ }
+ }
+
+ template <typename Stream>
+ void Unserialize(Stream& s)
+ {
+ std::vector<uint64_t> tmp;
+ s >> blockhash >> VectorApply<CompactSizeWrapper>(tmp);
@sipa

sipa Jul 10, 2017 edited

Owner

For ease of implementation, deserialization first happens into a std::vector<uint64_t>, and is then converted. This means a temporary is created and allocated, which is an overhead that the old implementation didn't have.

+
+ indexes.resize(tmp.size());
+ uint16_t offset = 0;
+ for (size_t j = 0; j < tmp.size(); j++) {
+ if (tmp[j] + uint64_t(offset) > std::numeric_limits<uint16_t>::max())
+ throw std::ios_base::failure("indexes overflowed 16 bits");
+ indexes[j] = tmp[j] + offset;
+ offset = indexes[j] + 1;
}
}
};
@@ -78,24 +90,9 @@ class BlockTransactions {
BlockTransactions(const BlockTransactionsRequest& req) :
blockhash(req.blockhash), txn(req.indexes.size()) {}
- ADD_SERIALIZE_METHODS;
-
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITE(blockhash);
- uint64_t txn_size = (uint64_t)txn.size();
- READWRITE(COMPACTSIZE(txn_size));
- if (ser_action.ForRead()) {
- size_t i = 0;
- while (txn.size() < txn_size) {
- txn.resize(std::min((uint64_t)(1000 + txn.size()), txn_size));
- for (; i < txn.size(); i++)
- READWRITE(REF(TransactionCompressor(txn[i])));
- }
- } else {
- for (size_t i = 0; i < txn.size(); i++)
- READWRITE(REF(TransactionCompressor(txn[i])));
- }
+ SERIALIZE_METHODS(obj)
+ {
+ READWRITE(obj.blockhash, VectorApply<TransactionCompressWrapper>(obj.txn));
}
};
@@ -106,16 +103,21 @@ struct PrefilledTransaction {
uint16_t index;
CTransactionRef tx;
- ADD_SERIALIZE_METHODS;
+ template<typename Stream>
+ void Serialize(Stream& s) const
+ {
+ s << COMPACTSIZE(index) << TransactionCompressor(tx);
+ }
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- uint64_t idx = index;
- READWRITE(COMPACTSIZE(idx));
+ template<typename Stream>
+ void Unserialize(Stream& s)
+ {
+ uint64_t idx;
+ s >> COMPACTSIZE(idx);
if (idx > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("index overflowed 16-bits");
index = idx;
- READWRITE(REF(TransactionCompressor(tx)));
+ s >> TransactionCompressor(tx);
}
};
@@ -154,40 +156,18 @@ class CBlockHeaderAndShortTxIDs {
size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); }
- ADD_SERIALIZE_METHODS;
-
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITE(header);
- READWRITE(nonce);
-
- uint64_t shorttxids_size = (uint64_t)shorttxids.size();
- READWRITE(COMPACTSIZE(shorttxids_size));
- if (ser_action.ForRead()) {
- size_t i = 0;
- while (shorttxids.size() < shorttxids_size) {
- shorttxids.resize(std::min((uint64_t)(1000 + shorttxids.size()), shorttxids_size));
- for (; i < shorttxids.size(); i++) {
- uint32_t lsb = 0; uint16_t msb = 0;
- READWRITE(lsb);
- READWRITE(msb);
- shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb);
- static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids serialization assumes 6-byte shorttxids");
- }
- }
- } else {
- for (size_t i = 0; i < shorttxids.size(); i++) {
- uint32_t lsb = shorttxids[i] & 0xffffffff;
- uint16_t msb = (shorttxids[i] >> 32) & 0xffff;
- READWRITE(lsb);
- READWRITE(msb);
- }
- }
-
- READWRITE(prefilledtxn);
+ template <typename Stream>
+ void Serialize(Stream& s) const
+ {
+ s << header << nonce << VectorApply<Uint48Wrapper>(shorttxids) << prefilledtxn;
+ }
- if (ser_action.ForRead())
- FillShortTxIDSelector();
+ template <typename Stream>
+ inline void Unserialize(Stream& s)
+ {
+ static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids serialization assumes 6-byte shorttxids");
+ s >> header >> nonce >> VectorApply<Uint48Wrapper>(shorttxids) >> prefilledtxn;
+ FillShortTxIDSelector();
}
};
View
@@ -70,15 +70,7 @@ class CBloomFilter
CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak, unsigned char nFlagsIn);
CBloomFilter() : isFull(true), isEmpty(false), nHashFuncs(0), nTweak(0), nFlags(0) {}
- ADD_SERIALIZE_METHODS;
-
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITE(vData);
- READWRITE(nHashFuncs);
- READWRITE(nTweak);
- READWRITE(nFlags);
- }
+ SERIALIZE_METHODS(obj) { READWRITE(obj.vData, obj.nHashFuncs, obj.nTweak, obj.nFlags); }
void insert(const std::vector<unsigned char>& vKey);
void insert(const COutPoint& outpoint);
View
@@ -39,17 +39,15 @@ class CBlockFileInfo
uint64_t nTimeFirst; //!< earliest time of block in file
uint64_t nTimeLast; //!< latest time of block in file
- ADD_SERIALIZE_METHODS;
-
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITE(VARINT(nBlocks));
- READWRITE(VARINT(nSize));
- READWRITE(VARINT(nUndoSize));
- READWRITE(VARINT(nHeightFirst));
- READWRITE(VARINT(nHeightLast));
- READWRITE(VARINT(nTimeFirst));
- READWRITE(VARINT(nTimeLast));
+ SERIALIZE_METHODS(obj)
+ {
+ READWRITE(VARINT(obj.nBlocks));
+ READWRITE(VARINT(obj.nSize));
+ READWRITE(VARINT(obj.nUndoSize));
+ READWRITE(VARINT(obj.nHeightFirst));
+ READWRITE(VARINT(obj.nHeightLast));
+ READWRITE(VARINT(obj.nTimeFirst));
+ READWRITE(VARINT(obj.nTimeLast));
}
void SetNull() {
@@ -87,13 +85,7 @@ struct CDiskBlockPos
int nFile;
unsigned int nPos;
- ADD_SERIALIZE_METHODS;
-
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
- READWRITE(VARINT(nFile));
- READWRITE(VARINT(nPos));
- }
+ SERIALIZE_METHODS(obj) { READWRITE(VARINT(obj.nFile), VARINT(obj.nPos)); }
CDiskBlockPos() {
SetNull();
@@ -380,31 +372,29 @@ class CDiskBlockIndex : public CBlockIndex
hashPrev = (pprev ? pprev->GetBlockHash() : uint256());
}
- ADD_SERIALIZE_METHODS;
-
- template <typename Stream, typename Operation>
- inline void SerializationOp(Stream& s, Operation ser_action) {
+ SERIALIZE_METHODS(obj)
+ {
int _nVersion = s.GetVersion();
if (!(s.GetType() & SER_GETHASH))
READWRITE(VARINT(_nVersion));
- READWRITE(VARINT(nHeight));
- READWRITE(VARINT(nStatus));
- READWRITE(VARINT(nTx));
- if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO))
- READWRITE(VARINT(nFile));
- if (nStatus & BLOCK_HAVE_DATA)
- READWRITE(VARINT(nDataPos));
- if (nStatus & BLOCK_HAVE_UNDO)
- READWRITE(VARINT(nUndoPos));
+ READWRITE(VARINT(obj.nHeight));
+ READWRITE(VARINT(obj.nStatus));
+ READWRITE(VARINT(obj.nTx));
+ if (obj.nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO))
+ READWRITE(VARINT(obj.nFile));
+ if (obj.nStatus & BLOCK_HAVE_DATA)
+ READWRITE(VARINT(obj.nDataPos));
+ if (obj.nStatus & BLOCK_HAVE_UNDO)
+ READWRITE(VARINT(obj.nUndoPos));
// block header
- READWRITE(this->nVersion);
- READWRITE(hashPrev);
- READWRITE(hashMerkleRoot);
- READWRITE(nTime);
- READWRITE(nBits);
- READWRITE(nNonce);
+ READWRITE(obj.nVersion);
+ READWRITE(obj.hashPrev);
+ READWRITE(obj.hashMerkleRoot);
+ READWRITE(obj.nTime);
+ READWRITE(obj.nBits);
+ READWRITE(obj.nNonce);
}
uint256 GetBlockHash() const
View
@@ -24,7 +24,7 @@
*
* Serialized format:
* - VARINT((coinbase ? 1 : 0) | (height << 1))
- * - the non-spent CTxOut (via CTxOutCompressor)
+ * - the non-spent CTxOut (via TxOutCompress)
*/
class Coin
{
@@ -60,7 +60,7 @@ class Coin
assert(!IsSpent());
uint32_t code = nHeight * 2 + fCoinBase;
::Serialize(s, VARINT(code));
- ::Serialize(s, CTxOutCompressor(REF(out)));
+ ::Serialize(s, TxOutCompress(out));
}
template<typename Stream>
@@ -69,7 +69,7 @@ class Coin
::Unserialize(s, VARINT(code));
nHeight = code >> 1;
fCoinBase = code & 1;
- ::Unserialize(s, REF(CTxOutCompressor(out)));
+ ::Unserialize(s, TxOutCompress(out));
}
bool IsSpent() const {
Oops, something went wrong.