Permalink
Browse files

Merge #8173: Use SipHash for node eviction (cont'd)

eebc232 test: Add more test vectors for siphash (Wladimir J. van der Laan)
8884830 Use C++11 thread-safe static initializers (Pieter Wuille)
c31b24f Use 64-bit SipHash of netgroups in eviction (Pieter Wuille)
9bf156b Support SipHash with arbitrary byte writes (Pieter Wuille)
053930f Avoid recalculating vchKeyedNetGroup in eviction logic. (Patrick Strateman)
  • Loading branch information...
sipa committed Jun 8, 2016
2 parents cd0c513 + eebc232 commit 4286f43025149cf44207c3ad98e4a1f068520ada
Showing with 152 additions and 62 deletions.
  1. +1 −5 src/coins.cpp
  2. +1 −1 src/coins.h
  3. +36 −3 src/hash.cpp
  4. +20 −1 src/hash.h
  5. +2 −5 src/main.cpp
  6. +26 −42 src/net.cpp
  7. +5 −1 src/net.h
  8. +61 −4 src/test/hash_tests.cpp
View
@@ -56,11 +56,7 @@ void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; }
bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); }
CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); }
SaltedTxidHasher::SaltedTxidHasher()
{
GetRandBytes((unsigned char*)&k0, sizeof(k0));
GetRandBytes((unsigned char*)&k1, sizeof(k1));
}
SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {}
CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { }
View
@@ -269,7 +269,7 @@ class SaltedTxidHasher
{
private:
/** Salt */
uint64_t k0, k1;
const uint64_t k0, k1;
public:
SaltedTxidHasher();
View
@@ -100,12 +100,15 @@ CSipHasher::CSipHasher(uint64_t k0, uint64_t k1)
v[2] = 0x6c7967656e657261ULL ^ k0;
v[3] = 0x7465646279746573ULL ^ k1;
count = 0;
tmp = 0;
}
CSipHasher& CSipHasher::Write(uint64_t data)
{
uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
assert(count % 8 == 0);
v3 ^= data;
SIPROUND;
SIPROUND;
@@ -116,18 +119,48 @@ CSipHasher& CSipHasher::Write(uint64_t data)
v[2] = v2;
v[3] = v3;
count++;
count += 8;
return *this;
}
CSipHasher& CSipHasher::Write(const unsigned char* data, size_t size)
{
uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
uint64_t t = tmp;
int c = count;
while (size--) {
t |= ((uint64_t)(*(data++))) << (8 * (c % 8));
c++;
if ((c & 7) == 0) {
v3 ^= t;
SIPROUND;
SIPROUND;
v0 ^= t;
t = 0;
}
}
v[0] = v0;
v[1] = v1;
v[2] = v2;
v[3] = v3;
count = c;
tmp = t;
return *this;
}
uint64_t CSipHasher::Finalize() const
{
uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
v3 ^= ((uint64_t)count) << 59;
uint64_t t = tmp | (((uint64_t)count) << 56);
v3 ^= t;
SIPROUND;
SIPROUND;
v0 ^= ((uint64_t)count) << 59;
v0 ^= t;
v2 ^= 0xFF;
SIPROUND;
SIPROUND;
View
@@ -171,19 +171,38 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char
void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]);
/** SipHash-2-4, using a uint64_t-based (rather than byte-based) interface */
/** SipHash-2-4 */
class CSipHasher
{
private:
uint64_t v[4];
uint64_t tmp;
int count;
public:
/** Construct a SipHash calculator initialized with 128-bit key (k0, k1) */
CSipHasher(uint64_t k0, uint64_t k1);
/** Hash a 64-bit integer worth of data
* It is treated as if this was the little-endian interpretation of 8 bytes.
* This function can only be used when a multiple of 8 bytes have been written so far.
*/
CSipHasher& Write(uint64_t data);
/** Hash arbitrary bytes. */
CSipHasher& Write(const unsigned char* data, size_t size);
/** Compute the 64-bit SipHash-2-4 of the data written so far. The object remains untouched. */
uint64_t Finalize() const;
};
/** Optimized SipHash-2-4 implementation for uint256.
*
* It is identical to:
* SipHasher(k0, k1)
* .Write(val.GetUint64(0))
* .Write(val.GetUint64(1))
* .Write(val.GetUint64(2))
* .Write(val.GetUint64(3))
* .Finalize()
*/
uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val);
#endif // BITCOIN_HASH_H
View
@@ -4783,11 +4783,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
LOCK(cs_vNodes);
// Use deterministic randomness to send to the same nodes for 24 hours
// at a time so the addrKnowns of the chosen nodes prevent repeats
static uint64_t salt0 = 0, salt1 = 0;
while (salt0 == 0 && salt1 == 0) {
GetRandBytes((unsigned char*)&salt0, sizeof(salt0));
GetRandBytes((unsigned char*)&salt1, sizeof(salt1));
}
static const uint64_t salt0 = GetRand(std::numeric_limits<uint64_t>::max());
static const uint64_t salt1 = GetRand(std::numeric_limits<uint64_t>::max());
uint64_t hashAddr = addr.GetHash();
multimap<uint64_t, CNode*> mapMix;
const CSipHasher hasher = CSipHasher(salt0, salt1).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60));
View
@@ -14,6 +14,7 @@
#include "clientversion.h"
#include "consensus/consensus.h"
#include "crypto/common.h"
#include "crypto/sha256.h"
#include "hash.h"
#include "primitives/transaction.h"
#include "scheduler.h"
@@ -838,6 +839,7 @@ struct NodeEvictionCandidate
int64_t nTimeConnected;
int64_t nMinPingUsecTime;
CAddress addr;
uint64_t nKeyedNetGroup;
};
static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b)
@@ -850,36 +852,8 @@ static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, cons
return a.nTimeConnected > b.nTimeConnected;
}
class CompareNetGroupKeyed
{
std::vector<unsigned char> vchSecretKey;
public:
CompareNetGroupKeyed()
{
vchSecretKey.resize(32, 0);
GetRandBytes(vchSecretKey.data(), vchSecretKey.size());
}
bool operator()(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b)
{
std::vector<unsigned char> vchGroupA, vchGroupB;
CSHA256 hashA, hashB;
std::vector<unsigned char> vchA(32), vchB(32);
vchGroupA = a.addr.GetGroup();
vchGroupB = b.addr.GetGroup();
hashA.Write(begin_ptr(vchGroupA), vchGroupA.size());
hashB.Write(begin_ptr(vchGroupB), vchGroupB.size());
hashA.Write(begin_ptr(vchSecretKey), vchSecretKey.size());
hashB.Write(begin_ptr(vchSecretKey), vchSecretKey.size());
hashA.Finalize(begin_ptr(vchA));
hashB.Finalize(begin_ptr(vchB));
return vchA < vchB;
}
static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) {
return a.nKeyedNetGroup < b.nKeyedNetGroup;
};
/** Try to find a connection to evict when the node is full.
@@ -902,7 +876,7 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) {
continue;
if (node->fDisconnect)
continue;
NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, node->addr};
NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, node->addr, node->nKeyedNetGroup};
vEvictionCandidates.push_back(candidate);
}
}
@@ -912,9 +886,8 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) {
// Protect connections with certain characteristics
// Deterministically select 4 peers to protect by netgroup.
// An attacker cannot predict which netgroups will be protected.
static CompareNetGroupKeyed comparerNetGroupKeyed;
std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), comparerNetGroupKeyed);
// An attacker cannot predict which netgroups will be protected
std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNetGroupKeyed);
vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end());
if (vEvictionCandidates.empty()) return false;
@@ -935,24 +908,24 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) {
// Identify the network group with the most connections and youngest member.
// (vEvictionCandidates is already sorted by reverse connect time)
std::vector<unsigned char> naMostConnections;
uint64_t naMostConnections;
unsigned int nMostConnections = 0;
int64_t nMostConnectionsTime = 0;
std::map<std::vector<unsigned char>, std::vector<NodeEvictionCandidate> > mapAddrCounts;
std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapAddrCounts;
BOOST_FOREACH(const NodeEvictionCandidate &node, vEvictionCandidates) {
mapAddrCounts[node.addr.GetGroup()].push_back(node);
int64_t grouptime = mapAddrCounts[node.addr.GetGroup()][0].nTimeConnected;
size_t groupsize = mapAddrCounts[node.addr.GetGroup()].size();
mapAddrCounts[node.nKeyedNetGroup].push_back(node);
int64_t grouptime = mapAddrCounts[node.nKeyedNetGroup][0].nTimeConnected;
size_t groupsize = mapAddrCounts[node.nKeyedNetGroup].size();
if (groupsize > nMostConnections || (groupsize == nMostConnections && grouptime > nMostConnectionsTime)) {
nMostConnections = groupsize;
nMostConnectionsTime = grouptime;
naMostConnections = node.addr.GetGroup();
naMostConnections = node.nKeyedNetGroup;
}
}
// Reduce to the network group with the most connections
vEvictionCandidates = mapAddrCounts[naMostConnections];
vEvictionCandidates = std::move(mapAddrCounts[naMostConnections]);
// Do not disconnect peers if there is only one unprotected connection from their network group.
// This step excessively favors netgroup diversity, and should be removed once more protective criteria are established.
@@ -2346,6 +2319,8 @@ unsigned int SendBufferSize() { return 1000*GetArg("-maxsendbuffer", DEFAULT_MAX
CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNameIn, bool fInboundIn) :
ssSend(SER_NETWORK, INIT_PROTO_VERSION),
addr(addrIn),
nKeyedNetGroup(CalculateKeyedNetGroup(addrIn)),
addrKnown(5000, 0.001),
filterInventoryKnown(50000, 0.000001)
{
@@ -2358,7 +2333,6 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa
nRecvBytes = 0;
nTimeConnected = GetTime();
nTimeOffset = 0;
addr = addrIn;
addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn;
nVersion = 0;
strSubVer = "";
@@ -2625,3 +2599,13 @@ bool CBanDB::Read(banmap_t& banSet)
int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds) {
return nNow + (int64_t)(log1p(GetRand(1ULL << 48) * -0.0000000000000035527136788 /* -1/2^48 */) * average_interval_seconds * -1000000.0 + 0.5);
}
/* static */ uint64_t CNode::CalculateKeyedNetGroup(const CAddress& ad)
{
static const uint64_t k0 = GetRand(std::numeric_limits<uint64_t>::max());
static const uint64_t k1 = GetRand(std::numeric_limits<uint64_t>::max());
std::vector<unsigned char> vchNetGroup(ad.GetGroup());
return CSipHasher(k0, k1).Write(&vchNetGroup[0], vchNetGroup.size()).Finalize();
}
View
@@ -335,7 +335,7 @@ class CNode
int64_t nLastRecv;
int64_t nTimeConnected;
int64_t nTimeOffset;
CAddress addr;
const CAddress addr;
std::string addrName;
CService addrLocal;
int nVersion;
@@ -362,6 +362,8 @@ class CNode
CBloomFilter* pfilter;
int nRefCount;
NodeId id;
const uint64_t nKeyedNetGroup;
protected:
// Denial-of-service detection/prevention
@@ -450,6 +452,8 @@ class CNode
CNode(const CNode&);
void operator=(const CNode&);
static uint64_t CalculateKeyedNetGroup(const CAddress& ad);
public:
NodeId GetId() const {
View
@@ -47,24 +47,81 @@ BOOST_AUTO_TEST_CASE(murmurhash3)
#undef T
}
/*
SipHash-2-4 output with
k = 00 01 02 ...
and
in = (empty string)
in = 00 (1 byte)
in = 00 01 (2 bytes)
in = 00 01 02 (3 bytes)
...
in = 00 01 02 ... 3e (63 bytes)
from: https://131002.net/siphash/siphash24.c
*/
uint64_t siphash_4_2_testvec[] = {
0x726fdb47dd0e0e31, 0x74f839c593dc67fd, 0x0d6c8009d9a94f5a, 0x85676696d7fb7e2d,
0xcf2794e0277187b7, 0x18765564cd99a68d, 0xcbc9466e58fee3ce, 0xab0200f58b01d137,
0x93f5f5799a932462, 0x9e0082df0ba9e4b0, 0x7a5dbbc594ddb9f3, 0xf4b32f46226bada7,
0x751e8fbc860ee5fb, 0x14ea5627c0843d90, 0xf723ca908e7af2ee, 0xa129ca6149be45e5,
0x3f2acc7f57c29bdb, 0x699ae9f52cbe4794, 0x4bc1b3f0968dd39c, 0xbb6dc91da77961bd,
0xbed65cf21aa2ee98, 0xd0f2cbb02e3b67c7, 0x93536795e3a33e88, 0xa80c038ccd5ccec8,
0xb8ad50c6f649af94, 0xbce192de8a85b8ea, 0x17d835b85bbb15f3, 0x2f2e6163076bcfad,
0xde4daaaca71dc9a5, 0xa6a2506687956571, 0xad87a3535c49ef28, 0x32d892fad841c342,
0x7127512f72f27cce, 0xa7f32346f95978e3, 0x12e0b01abb051238, 0x15e034d40fa197ae,
0x314dffbe0815a3b4, 0x027990f029623981, 0xcadcd4e59ef40c4d, 0x9abfd8766a33735c,
0x0e3ea96b5304a7d0, 0xad0c42d6fc585992, 0x187306c89bc215a9, 0xd4a60abcf3792b95,
0xf935451de4f21df2, 0xa9538f0419755787, 0xdb9acddff56ca510, 0xd06c98cd5c0975eb,
0xe612a3cb9ecba951, 0xc766e62cfcadaf96, 0xee64435a9752fe72, 0xa192d576b245165a,
0x0a8787bf8ecb74b2, 0x81b3e73d20b49b6f, 0x7fa8220ba3b2ecea, 0x245731c13ca42499,
0xb78dbfaf3a8d83bd, 0xea1ad565322a1a0b, 0x60e61c23a3795013, 0x6606d7e446282b93,
0x6ca4ecb15c5f91e1, 0x9f626da15c9625f3, 0xe51b38608ef25f57, 0x958a324ceb064572
};
BOOST_AUTO_TEST_CASE(siphash)
{
CSipHasher hasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x726fdb47dd0e0e31ull);
hasher.Write(0x0706050403020100ULL);
static const unsigned char t0[1] = {0};
hasher.Write(t0, 1);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x74f839c593dc67fdull);
static const unsigned char t1[7] = {1,2,3,4,5,6,7};
hasher.Write(t1, 7);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x93f5f5799a932462ull);
hasher.Write(0x0F0E0D0C0B0A0908ULL);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x3f2acc7f57c29bdbull);
hasher.Write(0x1716151413121110ULL);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0xb8ad50c6f649af94ull);
hasher.Write(0x1F1E1D1C1B1A1918ULL);
static const unsigned char t2[2] = {16,17};
hasher.Write(t2, 2);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x4bc1b3f0968dd39cull);
static const unsigned char t3[9] = {18,19,20,21,22,23,24,25,26};
hasher.Write(t3, 9);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x2f2e6163076bcfadull);
static const unsigned char t4[5] = {27,28,29,30,31};
hasher.Write(t4, 5);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x7127512f72f27cceull);
hasher.Write(0x2726252423222120ULL);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x0e3ea96b5304a7d0ull);
hasher.Write(0x2F2E2D2C2B2A2928ULL);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0xe612a3cb9ecba951ull);
BOOST_CHECK_EQUAL(SipHashUint256(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL, uint256S("1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100")), 0x7127512f72f27cceull);
// Check test vectors from spec, one byte at a time
CSipHasher hasher2(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL);
for (uint8_t x=0; x<ARRAYLEN(siphash_4_2_testvec); ++x)
{
BOOST_CHECK_EQUAL(hasher2.Finalize(), siphash_4_2_testvec[x]);
hasher2.Write(&x, 1);
}
// Check test vectors from spec, eight bytes at a time
CSipHasher hasher3(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL);
for (uint8_t x=0; x<ARRAYLEN(siphash_4_2_testvec); x+=8)
{
BOOST_CHECK_EQUAL(hasher3.Finalize(), siphash_4_2_testvec[x]);
hasher3.Write(uint64_t(x)|(uint64_t(x+1)<<8)|(uint64_t(x+2)<<16)|(uint64_t(x+3)<<24)|
(uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56));
}
}
BOOST_AUTO_TEST_SUITE_END()

0 comments on commit 4286f43

Please sign in to comment.