Skip to content

Commit

Permalink
Use precompiled structs for header parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@erasmospunk
erasmospunk committed Aug 13, 2018
1 parent 5a01312 commit 31dbbe0
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 136 deletions.
205 changes: 70 additions & 135 deletions electrumx/lib/coins.py
View file
Expand Up @@ -70,6 +70,9 @@ class Coin(object):
DESERIALIZER = lib_tx.Deserializer
DAEMON = daemon.Daemon
BLOCK_PROCESSOR = block_proc.BlockProcessor
HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'timestamp',
'bits', 'nonce')
HEADER_UNPACK = struct.Struct('< I 32s 32s I I I').unpack_from
MEMPOOL_HISTOGRAM_REFRESH_SECS = 500
XPUB_VERBYTES = bytes('????', 'utf-8')
XPRV_VERBYTES = bytes('????', 'utf-8')
Expand Down Expand Up @@ -296,18 +299,13 @@ def decimal_value(cls, value):

@classmethod
def electrum_header(cls, header, height):
version, = struct.unpack('<I', header[:4])
timestamp, bits, nonce = struct.unpack('<III', header[68:80])

return {
'block_height': height,
'version': version,
'prev_block_hash': hash_to_hex_str(header[4:36]),
'merkle_root': hash_to_hex_str(header[36:68]),
'timestamp': timestamp,
'bits': bits,
'nonce': nonce,
}
h = dict(zip(cls.HEADER_VALUES, cls.HEADER_UNPACK(header)))
# Add the height that is not present in the header itself
h['block_height'] = height
# Convert bytes to str
h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash'])
h['merkle_root'] = hash_to_hex_str(h['merkle_root'])
return h


class AuxPowMixin(object):
Expand All @@ -330,21 +328,21 @@ class EquihashMixin(object):
STATIC_BLOCK_HEADERS = False
BASIC_HEADER_SIZE = 140 # Excluding Equihash solution
DESERIALIZER = lib_tx.DeserializerEquihash
HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'reserved',
'timestamp', 'bits', 'nonce')
HEADER_UNPACK = struct.Struct('< I 32s 32s 32s I I 32s').unpack_from

@classmethod
def electrum_header(cls, header, height):
version, = struct.unpack('<I', header[:4])
timestamp, bits = struct.unpack('<II', header[100:108])

return {
'block_height': height,
'version': version,
'prev_block_hash': hash_to_hex_str(header[4:36]),
'merkle_root': hash_to_hex_str(header[36:68]),
'timestamp': timestamp,
'bits': bits,
'nonce': hash_to_hex_str(header[108:140]),
}
h = dict(zip(cls.HEADER_VALUES, cls.HEADER_UNPACK(header)))
# Add the height that is not present in the header itself
h['block_height'] = height
# Convert bytes to str
h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash'])
h['merkle_root'] = hash_to_hex_str(h['merkle_root'])
h['reserved'] = hash_to_hex_str(h['reserved'])
h['nonce'] = hash_to_hex_str(h['nonce'])
return h

@classmethod
def block_header(cls, block, height):
Expand All @@ -365,7 +363,7 @@ def header_hash(cls, header):
import scrypt
cls.HEADER_HASH = lambda x: scrypt.hash(x, x, 1024, 1, 1, 32)

version, = struct.unpack('<I', header[:4])
version, = util.unpack_le_uint32_from(header)
if version > 6:
return super().header_hash(header)
else:
Expand Down Expand Up @@ -475,27 +473,17 @@ class BitcoinGold(EquihashMixin, BitcoinMixin, Coin):
@classmethod
def header_hash(cls, header):
'''Given a header return hash'''
height, = struct.unpack('<I', header[68:72])

height, = util.unpack_le_uint32_from(header, 68)
if height >= cls.FORK_HEIGHT:
return double_sha256(header)
else:
return double_sha256(header[:68] + header[100:112])

@classmethod
def electrum_header(cls, header, height):
h = dict(
block_height=height,
version=struct.unpack('<I', header[:4])[0],
prev_block_hash=hash_to_hex_str(header[4:36]),
merkle_root=hash_to_hex_str(header[36:68]),
timestamp=struct.unpack('<I', header[100:104])[0],
reserved=hash_to_hex_str(header[72:100]),
bits=struct.unpack('<I', header[104:108])[0],
nonce=hash_to_hex_str(header[108:140]),
solution=hash_to_hex_str(header[140:])
)

h = super().electrum_header(header, height)
h['reserved'] = hash_to_hex_str(header[72:100])
h['solution'] = hash_to_hex_str(header[140:])
return h


Expand Down Expand Up @@ -948,6 +936,9 @@ class FairCoin(Coin):
GENESIS_HASH = ('beed44fa5e96150d95d56ebd5d262578'
'1825a9407a5215dd7eda723373a0a1d7')
BASIC_HEADER_SIZE = 108
HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root',
'payload_hash', 'timestamp', 'creatorId')
HEADER_UNPACK = struct.Struct('< I 32s 32s 32s I I').unpack_from
TX_COUNT = 505
TX_COUNT_HEIGHT = 470
TX_PER_BLOCK = 1
Expand All @@ -968,17 +959,9 @@ def block(cls, raw_block, height):

@classmethod
def electrum_header(cls, header, height):
version, = struct.unpack('<I', header[:4])
timestamp, creatorId = struct.unpack('<II', header[100:108])
return {
'block_height': height,
'version': version,
'prev_block_hash': hash_to_hex_str(header[4:36]),
'merkle_root': hash_to_hex_str(header[36:68]),
'payload_hash': hash_to_hex_str(header[68:100]),
'timestamp': timestamp,
'creatorId': creatorId,
}
h = super().electrum_header(header, height)
h['payload_hash'] = hash_to_hex_str(h['payload_hash'])
return h


class Zcash(EquihashMixin, Coin):
Expand Down Expand Up @@ -1031,21 +1014,10 @@ class SnowGem(EquihashMixin, Coin):

@classmethod
def electrum_header(cls, header, height):
version, = struct.unpack('<I', header[:4])
timestamp, bits = struct.unpack('<II', header[100:108])

return {
'block_height': height,
'version': version,
'prev_block_hash': hash_to_hex_str(header[4:36]),
'merkle_root': hash_to_hex_str(header[36:68]),
'hash_reserved': hash_to_hex_str(header[68:100]),
'timestamp': timestamp,
'bits': bits,
'nonce': hash_to_hex_str(header[108:140]),
'n_solution': base64.b64encode(lib_tx.Deserializer(
header, start=140)._read_varbytes()).decode('utf8')
}
h = super().electrum_header(header, height)
h['n_solution'] = base64.b64encode(lib_tx.Deserializer(
header, start=140)._read_varbytes()).decode('utf8')
return h


class BitcoinZ(EquihashMixin, Coin):
Expand Down Expand Up @@ -1670,7 +1642,7 @@ def header_hash(cls, header):
header_to_be_hashed = header[:cls.BASIC_HEADER_SIZE]
# New block header format has some extra flags in the end
if len(header) == cls.HEADER_SIZE_POST_FORK:
flags, = struct.unpack('<I', header[-4:])
flags, = util.unpack_le_uint32_from(header, len(header) - 4)
# Proof of work blocks have special serialization
if flags & cls.BLOCK_PROOF_OF_STAKE != 0:
header_to_be_hashed += cls.BLOCK_PROOF_OF_STAKE_FLAGS
Expand Down Expand Up @@ -1704,7 +1676,13 @@ class Decred(Coin):
hash_fn=lib_tx.DeserializerDecred.blake256d)
DECODE_CHECK = partial(Base58.decode_check,
hash_fn=lib_tx.DeserializerDecred.blake256d)
HEADER_UNPACK = struct.Struct('<i32s32s32sH6sHBBIIQIIII32sI').unpack_from
HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'stake_root',
'vote_bits', 'final_state', 'voters', 'fresh_stake',
'revocations', 'pool_size', 'bits', 'sbits',
'block_height', 'size', 'timestamp', 'nonce',
'extra_data', 'stake_version')
HEADER_UNPACK = struct.Struct(
'< i 32s 32s 32s H 6s H B B I I Q I I I I 32s I').unpack_from
TX_COUNT = 4629388
TX_COUNT_HEIGHT = 260628
TX_PER_BLOCK = 17
Expand All @@ -1726,20 +1704,10 @@ def block(cls, raw_block, height):

@classmethod
def electrum_header(cls, header, height):
labels = ('version', 'prev_block_hash', 'merkle_root', 'stake_root',
'vote_bits', 'final_state', 'voters', 'fresh_stake',
'revocations', 'pool_size', 'bits', 'sbits', 'block_height',
'size', 'timestamp', 'nonce', 'extra_data', 'stake_version')
values = cls.HEADER_UNPACK(header)
h = dict(zip(labels, values))

# Convert some values
assert h['block_height'] == height
h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash'])
h['merkle_root'] = hash_to_hex_str(h['merkle_root'])
h = super().electrum_header(header, height)
h['stake_root'] = hash_to_hex_str(h['stake_root'])
h['final_state'] = h['final_state'].hex()
h['extra_data'] = h['extra_data'].hex()
h['final_state'] = hash_to_hex_str(h['final_state'])
h['extra_data'] = hash_to_hex_str(h['extra_data'])
return h


Expand Down Expand Up @@ -1799,7 +1767,7 @@ def header_hash(cls, header):
Need to download `xevan_hash` module
Source code: https://github.com/xuez/xuez
'''
version, = struct.unpack('<I', header[:4])
version, = util.unpack_le_uint32_from(header)

import xevan_hash

Expand All @@ -1810,29 +1778,10 @@ def header_hash(cls, header):

@classmethod
def electrum_header(cls, header, height):
version, = struct.unpack('<I', header[:4])
timestamp, bits, nonce = struct.unpack('<III', header[68:80])
if version == 1:
return {
'block_height': height,
'version': version,
'prev_block_hash': hash_to_hex_str(header[4:36]),
'merkle_root': hash_to_hex_str(header[36:68]),
'timestamp': timestamp,
'bits': bits,
'nonce': nonce,
}
else:
return {
'block_height': height,
'version': version,
'prev_block_hash': hash_to_hex_str(header[4:36]),
'merkle_root': hash_to_hex_str(header[36:68]),
'timestamp': timestamp,
'bits': bits,
'nonce': nonce,
'nAccumulatorCheckpoint': hash_to_hex_str(header[80:112]),
}
h = super().electrum_header(header, height)
if h['version'] > 1:
h['nAccumulatorCheckpoint'] = hash_to_hex_str(header[80:])
return h


class Pac(Coin):
Expand Down Expand Up @@ -1998,35 +1947,7 @@ def header_hash(cls, header):
return x11_hash.getPoWHash(header)


class MinexcoinMixin(object):
STATIC_BLOCK_HEADERS = True
BASIC_HEADER_SIZE = 209
DESERIALIZER = lib_tx.DeserializerEquihash

@classmethod
def electrum_header(cls, header, height):
version, = struct.unpack('<I', header[:4])
timestamp, bits = struct.unpack('<II', header[100:108])

return {
'block_height': height,
'version': version,
'prev_block_hash': hash_to_hex_str(header[4:36]),
'merkle_root': hash_to_hex_str(header[36:68]),
'timestamp': timestamp,
'bits': bits,
'nonce': hash_to_hex_str(header[108:140]),
'solution': hash_to_hex_str(header[140:209]),
}

@classmethod
def block_header(cls, block, height):
'''Return the block header bytes'''
deserializer = cls.DESERIALIZER(block)
return deserializer.read_header(height, 140)


class Minexcoin(MinexcoinMixin, Coin):
class Minexcoin(EquihashMixin, Coin):
NAME = "Minexcoin"
SHORTNAME = "MNX"
NET = "mainnet"
Expand All @@ -2035,7 +1956,9 @@ class Minexcoin(MinexcoinMixin, Coin):
WIF_BYTE = bytes.fromhex("80")
GENESIS_HASH = ('490a36d9451a55ed197e34aca7414b35'
'd775baa4a8e896f1c577f65ce2d214cb')
DESERIALIZER = lib_tx.DeserializerEquihash
STATIC_BLOCK_HEADERS = True
BASIC_HEADER_SIZE = 209
HEADER_SIZE_NO_SOLUTION = 140
TX_COUNT = 327963
TX_COUNT_HEIGHT = 74495
TX_PER_BLOCK = 5
Expand All @@ -2046,6 +1969,18 @@ class Minexcoin(MinexcoinMixin, Coin):
'eu.minexpool.nl s t'
]

@classmethod
def electrum_header(cls, header, height):
h = super().electrum_header(header, height)
h['solution'] = hash_to_hex_str(header[cls.HEADER_SIZE_NO_SOLUTION:])
return h

@classmethod
def block_header(cls, block, height):
'''Return the block header bytes'''
deserializer = cls.DESERIALIZER(block)
return deserializer.read_header(height, cls.HEADER_SIZE_NO_SOLUTION)


class Groestlcoin(Coin):
NAME = "Groestlcoin"
Expand Down Expand Up @@ -2133,7 +2068,7 @@ def static_header_offset(cls, height):

@classmethod
def header_hash(cls, header):
version, = struct.unpack('<I', header[:4])
version, = util.unpack_le_uint32_from(header)
if version >= 4:
return super().header_hash(header)
else:
Expand Down
2 changes: 1 addition & 1 deletion tests/blocks/newyorkcoin_mainnet_3956926.json
View file
Expand Up @@ -7,7 +7,7 @@
"0300b2fe16c049641ec63f9eb435b9d773c847bc0185c927d5b8d87cd1ad3095"
],
"time": 1515308763,
"nonce": "1570457681",
"nonce": 1570457681,
"bits": "1b172a4e",
"previousblockhash": "10c91aefd6698f03a2820b8aa462738a5c1c21f4f975786a8eb1ba25e4b33af3",
"block": "01000000f33ab3e425bab18e6a7875f9f4211c5c8a7362a48a0b82a2038f69d6ef1ac9109530add17cd8b8d527c98501bc47c873d7b935b49e3fc61e6449c016feb20003dbc6515a4e2a171b51489b5d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff5203be603c062f503253482f04dfc6515afabe6d6dce67985c41a5f1ba5d526614e1744010b5e6a34d33c50ece5d420d75da15454e100000000000000008b8000274020000000c2f30324d515156434c59582f00000000010010a5d4e80000001976a914f0a150ec5709fae1d1814227b69cd1f0baf528c588ac00000000"
Expand Down
8 changes: 8 additions & 0 deletions tests/test_blocks.py
View file
Expand Up @@ -32,6 +32,7 @@

from electrumx.lib.coins import Coin
from electrumx.lib.hash import hex_str_to_hash
from electrumx.lib.util import pack_be_uint32

BLOCKS_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'blocks')
Expand Down Expand Up @@ -60,6 +61,13 @@ def test_block(block_details):

raw_block = unhexlify(block_info['block'])
block = coin.block(raw_block, block_info['height'])
h = coin.electrum_header(block.header, block_info['height'])
assert block_info['merkleroot'] == h['merkle_root']
assert block_info['time'] == h['timestamp']
assert block_info['previousblockhash'] == h['prev_block_hash']
assert block_info['height'] == h['block_height']
assert block_info['nonce'] == h['nonce']
assert block_info['bits'] == pack_be_uint32(h['bits']).hex()

assert coin.header_hash(block.header) == hex_str_to_hash(block_info['hash'])
assert (coin.header_prevhash(block.header)
Expand Down

0 comments on commit 31dbbe0

Please sign in to comment.