Implement CBloomFilter class

bitcoin/bloom.py

@@ -0,0 +1,103 @@
+
+#
+# bloom.py
+#
+# Distributed under the MIT/X11 software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+
+import struct
+import math
+from bitcoin.serialize import *
+from bitcoin.coredefs import *
+from bitcoin.core import *
+from bitcoin.hash import MurmurHash3
+
+class CBloomFilter(object):
+    # 20,000 items with fp rate < 0.1% or 10,000 items and <0.0001%
+    MAX_BLOOM_FILTER_SIZE = 36000
+    MAX_HASH_FUNCS = 50
+
+    UPDATE_NONE = 0
+    UPDATE_ALL = 1
+    UPDATE_P2PUBKEY_ONLY = 2
+    UPDATE_MASK = 3
+
+    def __init__(self, nElements, nFPRate, nTweak, nFlags):
+        """Create a new bloom filter
+
+        The filter will have a given false-positive rate when filled with the
+        given number of elements.
+
+        Note that if the given parameters will result in a filter outside the
+        bounds of the protocol limits, the filter created will be as close to
+        the given parameters as possible within the protocol limits. This will
+        apply if nFPRate is very low or nElements is unreasonably high.
+
+        nTweak is a constant which is added to the seed value passed to the
+        hash function It should generally always be a random value (and is
+        largely only exposed for unit testing)
+
+        nFlags should be one of the UPDATE_* enums (but not _MASK)
+        """
+        LN2SQUARED = 0.4804530139182014246671025263266649717305529515945455
+        LN2 = 0.6931471805599453094172321214581765680755001343602552
+        self.vData = bytearray(int(min(-1  / LN2SQUARED * nElements * math.log(nFPRate), self.MAX_BLOOM_FILTER_SIZE * 8) / 8))
+        self.nHashFuncs = int(min(len(self.vData) * 8 / nElements * LN2, self.MAX_HASH_FUNCS))
+        self.nTweak = nTweak
+        self.nFlags = nFlags
+
+    def bloom_hash(self, nHashNum, vDataToHash):
+        return MurmurHash3(((nHashNum * 0xFBA4C795) + self.nTweak) & 0xFFFFFFFF, vDataToHash) % (len(self.vData) * 8)
+
+    __bit_mask = bytearray([0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80])
+    def insert(self, elem):
+        """Insert an element in the filter.
+
+        elem may be a COutPoint or bytes
+        """
+        if isinstance(elem, COutPoint):
+            elem = elem.serialize()
+
+        if len(self.vData) == 1 and self.vData[0] == 0xff:
+            return
+
+        for i in range(0, self.nHashFuncs):
+            nIndex = self.bloom_hash(i, elem)
+            # Sets bit nIndex of vData
+            self.vData[nIndex >> 3] |= self.__bit_mask[7 & nIndex]
+
+    def contains(self, elem):
+        """Test if the filter contains an element
+
+        elem may be a COutPoint or bytes
+        """
+        if isinstance(elem, COutPoint):
+            elem = elem.serialize()
+
+        if len(self.vData) == 1 and self.vData[0] == 0xff:
+            return True
+
+        for i in range(0, self.nHashFuncs):
+            nIndex = self.bloom_hash(i, elem)
+            if not (self.vData[nIndex >> 3] & self.__bit_mask[7 & nIndex]):
+                return False
+        return True
+
+    def IsWithinSizeConstraints(self):
+        return len(self.vData) <= self.MAX_BLOOM_FILTER_SIZE and self.nHashFuncs <= self.MAX_HASH_FUNCS
+
+    def IsRelevantAndUpdate(tx, tx_hash):
+        # Not useful for a client, so not implemented yet.
+        raise NotImplementedError
+
+    __struct = struct.Struct("<IIB")
+    def deserialize(self, f):
+        self.vData = deser_string(f)
+        (self.nHashFuncs,
+         self.nTweak,
+         self.nFlags) = self.__struct.unpack(f.read(self.__struct.size))
+
+    def serialize(self):
+        r = ser_string(self.vData)
+        return r + self.__struct.pack(self.nHashFuncs, self.nTweak, self.nFlags)

bitcoin/hash.py

@@ -0,0 +1,71 @@
+
+#
+# hash.py
+#
+# Distributed under the MIT/X11 software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+
+import struct
+from bitcoin.serialize import *
+from bitcoin.coredefs import *
+from bitcoin.script import CScript
+
+def ROTL32(x, r):
+    assert x <= 0xFFFFFFFF
+    return ((x << r) & 0xFFFFFFFF) | (x >> (32 - r))
+
+def MurmurHash3(nHashSeed, vDataToHash):
+    """MurmurHash3 (x86_32)
+
+    Used for bloom filters. See http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp
+    """
+
+    assert nHashSeed <= 0xFFFFFFFF
+
+    h1 = nHashSeed
+    c1 = 0xcc9e2d51
+    c2 = 0x1b873593
+
+    # body
+    i = 0
+    while i < len(vDataToHash) - len(vDataToHash) % 4 \
+          and len(vDataToHash) - i >= 4:
+
+        k1 = struct.unpack("<L", vDataToHash[i:i+4])[0]
+
+        k1 = (k1 * c1) & 0xFFFFFFFF
+        k1 = ROTL32(k1, 15)
+        k1 = (k1 * c2) & 0xFFFFFFFF
+
+        h1 ^= k1
+        h1 = ROTL32(h1, 13)
+        h1 = (((h1*5) & 0xFFFFFFFF) + 0xe6546b64) & 0xFFFFFFFF
+
+        i += 4
+
+    # tail
+    k1 = 0
+    j = (len(vDataToHash) // 4) * 4
+    if len(vDataToHash) & 3 >= 3:
+        k1 ^= struct.unpack('<B', vDataToHash[j+2])[0] << 16
+    if len(vDataToHash) & 3 >= 2:
+        k1 ^= struct.unpack('<B', vDataToHash[j+1])[0] << 8
+    if len(vDataToHash) & 3 >= 1:
+        k1 ^= struct.unpack('<B', vDataToHash[j])[0]
+
+    k1 &= 0xFFFFFFFF
+    k1 = (k1 * c1) & 0xFFFFFFFF
+    k1 = ROTL32(k1, 15)
+    k1 = (k1 * c2) & 0xFFFFFFFF
+    h1 ^= k1
+
+    # finalization
+    h1 ^= len(vDataToHash) & 0xFFFFFFFF
+    h1 ^= (h1 & 0xFFFFFFFF) >> 16
+    h1 *= 0x85ebca6b
+    h1 ^= (h1 & 0xFFFFFFFF) >> 13
+    h1 *= 0xc2b2ae35
+    h1 ^= (h1 & 0xFFFFFFFF) >> 16
+
+    return h1 & 0xFFFFFFFF

bitcoin/tests/test_bloom.py

@@ -0,0 +1,67 @@
+# Distributed under the MIT/X11 software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+from __future__ import print_function
+
+import json
+import os
+import unittest
+
+from binascii import unhexlify
+
+from bitcoin.bloom import *
+
+class Test_CBloomFilter(unittest.TestCase):
+    def test_create_insert_serialize(self):
+        filter = CBloomFilter(3, 0.01, 0, CBloomFilter.UPDATE_ALL)
+
+        def T(elem):
+            """Filter contains elem"""
+            elem = unhexlify(elem)
+            filter.insert(elem)
+            self.assertTrue(filter.contains(elem))
+
+        def F(elem):
+            """Filter does not contain elem"""
+            elem = unhexlify(elem)
+            self.assertFalse(filter.contains(elem))
+
+        T('99108ad8ed9bb6274d3980bab5a85c048f0950c8')
+        F('19108ad8ed9bb6274d3980bab5a85c048f0950c8')
+        T('b5a2c786d9ef4658287ced5914b37a1b4aa32eee')
+        T('b9300670b4c5366e95b2699e8b18bc75e5f729c5')
+
+        self.assertEqual(filter.serialize(), unhexlify('03614e9b050000000000000001'))
+
+    def test_create_insert_serialize_with_tweak(self):
+        # Same test as bloom_create_insert_serialize, but we add a nTweak of 100
+        filter = CBloomFilter(3, 0.01, 2147483649, CBloomFilter.UPDATE_ALL)
+
+        def T(elem):
+            """Filter contains elem"""
+            elem = unhexlify(elem)
+            filter.insert(elem)
+            self.assertTrue(filter.contains(elem))
+
+        def F(elem):
+            """Filter does not contain elem"""
+            elem = unhexlify(elem)
+            self.assertFalse(filter.contains(elem))
+
+        T('99108ad8ed9bb6274d3980bab5a85c048f0950c8')
+        F('19108ad8ed9bb6274d3980bab5a85c048f0950c8')
+        T('b5a2c786d9ef4658287ced5914b37a1b4aa32eee')
+        T('b9300670b4c5366e95b2699e8b18bc75e5f729c5')
+
+        self.assertEqual(filter.serialize(), unhexlify('03ce4299050000000100008001'))
+
+    def test_bloom_create_insert_key(self):
+        filter = CBloomFilter(2, 0.001, 0, CBloomFilter.UPDATE_ALL)
+
+        pubkey = unhexlify('045B81F0017E2091E2EDCD5EECF10D5BDD120A5514CB3EE65B8447EC18BFC4575C6D5BF415E54E03B1067934A0F0BA76B01C6B9AB227142EE1D543764B69D901E0')
+        pubkeyhash = ser_uint160(Hash160(pubkey))
+
+        filter.insert(pubkey)
+        filter.insert(pubkeyhash)
+
+        self.assertEqual(filter.serialize(), unhexlify('038fc16b080000000000000001'))

bitcoin/tests/test_hash.py

@@ -0,0 +1,34 @@
+# Distributed under the MIT/X11 software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+from __future__ import print_function
+
+import json
+import os
+import unittest
+
+from binascii import unhexlify
+
+from bitcoin.hash import *
+
+class Test_MurmurHash3(unittest.TestCase):
+    def test(self):
+        def T(expected, seed, data):
+            self.assertEqual(MurmurHash3(seed, unhexlify(data)), expected)
+
+        T(0x00000000, 0x00000000, "");
+        T(0x6a396f08, 0xFBA4C795, "");
+        T(0x81f16f39, 0xffffffff, "");
+
+        T(0x514e28b7, 0x00000000, "00");
+        T(0xea3f0b17, 0xFBA4C795, "00");
+        T(0xfd6cf10d, 0x00000000, "ff");
+
+        T(0x16c6b7ab, 0x00000000, "0011");
+        T(0x8eb51c3d, 0x00000000, "001122");
+        T(0xb4471bf8, 0x00000000, "00112233");
+        T(0xe2301fa8, 0x00000000, "0011223344");
+        T(0xfc2e4a15, 0x00000000, "001122334455");
+        T(0xb074502c, 0x00000000, "00112233445566");
+        T(0x8034d2a0, 0x00000000, "0011223344556677");
+        T(0xb4698def, 0x00000000, "001122334455667788");