diff --git a/bitcoin/bloom.py b/bitcoin/bloom.py new file mode 100644 index 0000000..c786be1 --- /dev/null +++ b/bitcoin/bloom.py @@ -0,0 +1,103 @@ + +# +# bloom.py +# +# Distributed under the MIT/X11 software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# + +import struct +import math +from bitcoin.serialize import * +from bitcoin.coredefs import * +from bitcoin.core import * +from bitcoin.hash import MurmurHash3 + +class CBloomFilter(object): + # 20,000 items with fp rate < 0.1% or 10,000 items and <0.0001% + MAX_BLOOM_FILTER_SIZE = 36000 + MAX_HASH_FUNCS = 50 + + UPDATE_NONE = 0 + UPDATE_ALL = 1 + UPDATE_P2PUBKEY_ONLY = 2 + UPDATE_MASK = 3 + + def __init__(self, nElements, nFPRate, nTweak, nFlags): + """Create a new bloom filter + + The filter will have a given false-positive rate when filled with the + given number of elements. + + Note that if the given parameters will result in a filter outside the + bounds of the protocol limits, the filter created will be as close to + the given parameters as possible within the protocol limits. This will + apply if nFPRate is very low or nElements is unreasonably high. + + nTweak is a constant which is added to the seed value passed to the + hash function It should generally always be a random value (and is + largely only exposed for unit testing) + + nFlags should be one of the UPDATE_* enums (but not _MASK) + """ + LN2SQUARED = 0.4804530139182014246671025263266649717305529515945455 + LN2 = 0.6931471805599453094172321214581765680755001343602552 + self.vData = bytearray(int(min(-1 / LN2SQUARED * nElements * math.log(nFPRate), self.MAX_BLOOM_FILTER_SIZE * 8) / 8)) + self.nHashFuncs = int(min(len(self.vData) * 8 / nElements * LN2, self.MAX_HASH_FUNCS)) + self.nTweak = nTweak + self.nFlags = nFlags + + def bloom_hash(self, nHashNum, vDataToHash): + return MurmurHash3(((nHashNum * 0xFBA4C795) + self.nTweak) & 0xFFFFFFFF, vDataToHash) % (len(self.vData) * 8) + + __bit_mask = bytearray([0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80]) + def insert(self, elem): + """Insert an element in the filter. + + elem may be a COutPoint or bytes + """ + if isinstance(elem, COutPoint): + elem = elem.serialize() + + if len(self.vData) == 1 and self.vData[0] == 0xff: + return + + for i in range(0, self.nHashFuncs): + nIndex = self.bloom_hash(i, elem) + # Sets bit nIndex of vData + self.vData[nIndex >> 3] |= self.__bit_mask[7 & nIndex] + + def contains(self, elem): + """Test if the filter contains an element + + elem may be a COutPoint or bytes + """ + if isinstance(elem, COutPoint): + elem = elem.serialize() + + if len(self.vData) == 1 and self.vData[0] == 0xff: + return True + + for i in range(0, self.nHashFuncs): + nIndex = self.bloom_hash(i, elem) + if not (self.vData[nIndex >> 3] & self.__bit_mask[7 & nIndex]): + return False + return True + + def IsWithinSizeConstraints(self): + return len(self.vData) <= self.MAX_BLOOM_FILTER_SIZE and self.nHashFuncs <= self.MAX_HASH_FUNCS + + def IsRelevantAndUpdate(tx, tx_hash): + # Not useful for a client, so not implemented yet. + raise NotImplementedError + + __struct = struct.Struct("> (32 - r)) + +def MurmurHash3(nHashSeed, vDataToHash): + """MurmurHash3 (x86_32) + + Used for bloom filters. See http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp + """ + + assert nHashSeed <= 0xFFFFFFFF + + h1 = nHashSeed + c1 = 0xcc9e2d51 + c2 = 0x1b873593 + + # body + i = 0 + while i < len(vDataToHash) - len(vDataToHash) % 4 \ + and len(vDataToHash) - i >= 4: + + k1 = struct.unpack("= 3: + k1 ^= struct.unpack('= 2: + k1 ^= struct.unpack('= 1: + k1 ^= struct.unpack('> 16 + h1 *= 0x85ebca6b + h1 ^= (h1 & 0xFFFFFFFF) >> 13 + h1 *= 0xc2b2ae35 + h1 ^= (h1 & 0xFFFFFFFF) >> 16 + + return h1 & 0xFFFFFFFF diff --git a/bitcoin/tests/test_bloom.py b/bitcoin/tests/test_bloom.py new file mode 100644 index 0000000..c98762a --- /dev/null +++ b/bitcoin/tests/test_bloom.py @@ -0,0 +1,67 @@ +# Distributed under the MIT/X11 software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from __future__ import print_function + +import json +import os +import unittest + +from binascii import unhexlify + +from bitcoin.bloom import * + +class Test_CBloomFilter(unittest.TestCase): + def test_create_insert_serialize(self): + filter = CBloomFilter(3, 0.01, 0, CBloomFilter.UPDATE_ALL) + + def T(elem): + """Filter contains elem""" + elem = unhexlify(elem) + filter.insert(elem) + self.assertTrue(filter.contains(elem)) + + def F(elem): + """Filter does not contain elem""" + elem = unhexlify(elem) + self.assertFalse(filter.contains(elem)) + + T('99108ad8ed9bb6274d3980bab5a85c048f0950c8') + F('19108ad8ed9bb6274d3980bab5a85c048f0950c8') + T('b5a2c786d9ef4658287ced5914b37a1b4aa32eee') + T('b9300670b4c5366e95b2699e8b18bc75e5f729c5') + + self.assertEqual(filter.serialize(), unhexlify('03614e9b050000000000000001')) + + def test_create_insert_serialize_with_tweak(self): + # Same test as bloom_create_insert_serialize, but we add a nTweak of 100 + filter = CBloomFilter(3, 0.01, 2147483649, CBloomFilter.UPDATE_ALL) + + def T(elem): + """Filter contains elem""" + elem = unhexlify(elem) + filter.insert(elem) + self.assertTrue(filter.contains(elem)) + + def F(elem): + """Filter does not contain elem""" + elem = unhexlify(elem) + self.assertFalse(filter.contains(elem)) + + T('99108ad8ed9bb6274d3980bab5a85c048f0950c8') + F('19108ad8ed9bb6274d3980bab5a85c048f0950c8') + T('b5a2c786d9ef4658287ced5914b37a1b4aa32eee') + T('b9300670b4c5366e95b2699e8b18bc75e5f729c5') + + self.assertEqual(filter.serialize(), unhexlify('03ce4299050000000100008001')) + + def test_bloom_create_insert_key(self): + filter = CBloomFilter(2, 0.001, 0, CBloomFilter.UPDATE_ALL) + + pubkey = unhexlify('045B81F0017E2091E2EDCD5EECF10D5BDD120A5514CB3EE65B8447EC18BFC4575C6D5BF415E54E03B1067934A0F0BA76B01C6B9AB227142EE1D543764B69D901E0') + pubkeyhash = ser_uint160(Hash160(pubkey)) + + filter.insert(pubkey) + filter.insert(pubkeyhash) + + self.assertEqual(filter.serialize(), unhexlify('038fc16b080000000000000001')) diff --git a/bitcoin/tests/test_hash.py b/bitcoin/tests/test_hash.py new file mode 100644 index 0000000..5f71be0 --- /dev/null +++ b/bitcoin/tests/test_hash.py @@ -0,0 +1,34 @@ +# Distributed under the MIT/X11 software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from __future__ import print_function + +import json +import os +import unittest + +from binascii import unhexlify + +from bitcoin.hash import * + +class Test_MurmurHash3(unittest.TestCase): + def test(self): + def T(expected, seed, data): + self.assertEqual(MurmurHash3(seed, unhexlify(data)), expected) + + T(0x00000000, 0x00000000, ""); + T(0x6a396f08, 0xFBA4C795, ""); + T(0x81f16f39, 0xffffffff, ""); + + T(0x514e28b7, 0x00000000, "00"); + T(0xea3f0b17, 0xFBA4C795, "00"); + T(0xfd6cf10d, 0x00000000, "ff"); + + T(0x16c6b7ab, 0x00000000, "0011"); + T(0x8eb51c3d, 0x00000000, "001122"); + T(0xb4471bf8, 0x00000000, "00112233"); + T(0xe2301fa8, 0x00000000, "0011223344"); + T(0xfc2e4a15, 0x00000000, "001122334455"); + T(0xb074502c, 0x00000000, "00112233445566"); + T(0x8034d2a0, 0x00000000, "0011223344556677"); + T(0xb4698def, 0x00000000, "001122334455667788");