Skip to content
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
658 lines (516 sloc) 24.4 KB
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
copyright: (c) 2014-2015 by Halfmoon Labs, Inc.
copyright: (c) 2016 by
This file is part of Blockstack.
Blockstack is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Blockstack is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Blockstack. If not, see <>.
import simplejson
import random
import time
from client import (
connect_hostport, get_blockstackd_url, create_bitcoind_service_proxy,
get_blockstack_ops_hash_at, get_consensus_hashes, get_blockstack_transactions_at,
get_block_from_consensus, get_consensus_at)
import virtualchain
from virtualchain import SPVClient
from virtualchain.lib.hashing import is_hex
from config import (
get_bitcoin_opts, blockstack_magic_bytes
import json
log = virtualchain.get_logger('blockstackd-snv')
def get_default_proxy(blockstackd_url = None):
Connect to the blockstackd in the config file
if blockstackd_url is None:
blockstackd_url = get_blockstackd_url()
return connect_hostport(blockstackd_url)
def get_bitcoind_client():
Connect to the bitcoind node
bitcoind_opts = get_bitcoin_opts()
bitcoind_host = bitcoind_opts['bitcoind_server']
bitcoind_port = bitcoind_opts['bitcoind_port']
bitcoind_user = bitcoind_opts['bitcoind_user']
bitcoind_passwd = bitcoind_opts['bitcoind_passwd']
return create_bitcoind_service_proxy(bitcoind_user, bitcoind_passwd, server=bitcoind_host, port=bitcoind_port)
def txid_to_block_data(txid, bitcoind_proxy, proxy=None):
Given a txid, get its block's data.
Use SPV to verify the information we receive from the (untrusted)
bitcoind host.
@bitcoind_proxy must be a BitcoindConnection (from virtualchain.lib.session)
Return the (block hash, block data, txdata) on success
Return (None, None, None) on error
proxy = get_default_proxy() if proxy is None else proxy
timeout = 1.0
while True:
untrusted_tx_data = bitcoind_proxy.getrawtransaction(txid, 1)
untrusted_block_hash = untrusted_tx_data['blockhash']
untrusted_block_data = bitcoind_proxy.getblock(untrusted_block_hash)
except (OSError, IOError) as ie:
log.error('Network error; retrying...')
timeout = timeout * 2 + random.randint(0, timeout)
except Exception as e:
return None, None, None
bitcoind_opts = get_bitcoin_opts()
spv_headers_path = bitcoind_opts['bitcoind_spv_path']
# first, can we trust this block? is it in the SPV headers?
untrusted_block_header_hex = virtualchain.block_header_to_hex(
untrusted_block_data, untrusted_block_data['previousblockhash']
block_id = SPVClient.block_header_index(
if block_id < 0:
# bad header
log.error('Block header "{}" is not in the SPV headers ({})'.format(
untrusted_block_header_hex, spv_headers_path
return None, None, None
# block header is trusted. Is the transaction data consistent with it?
verified_block_header = virtualchain.block_verify(untrusted_block_data)
if not verified_block_header:
msg = (
'Block transaction IDs are not consistent '
'with the Merkle root of the trusted header'
return None, None, None
# verify block hash
verified_block_hash = virtualchain.block_header_verify(
untrusted_block_data, untrusted_block_data['previousblockhash'], untrusted_block_hash
if not verified_block_hash:
log.error('Block hash is not consistent with block header')
return None, None, None
# we trust the block hash, block data, and txids
block_hash = untrusted_block_hash
block_data = untrusted_block_data
tx_data = untrusted_tx_data
return block_hash, block_data, tx_data
def serial_number_to_tx(serial_number, bitcoind_proxy, proxy=None):
Convert a serial number into its transaction in the blockchain.
Use an untrusted bitcoind connection to get the list of transactions,
and use trusted SPV headers to ensure that the transaction obtained is on the main chain.
@bitcoind_proxy must be a BitcoindConnection (from virtualchain.lib.session)
Return the SPV-verified transaction object (as a dict) on success
Return None on error
proxy = get_default_proxy() if proxy is None else proxy
parts = serial_number.split('-')
block_id, tx_index = int(parts[0]), int(parts[1])
timeout = 1.0
while True:
block_hash = bitcoind_proxy.getblockhash(block_id)
block_data = bitcoind_proxy.getblock(block_hash)
except Exception as e:
log.error('Unable to obtain block data; retrying...')
timeout = timeout * 2 + random.random() * timeout
bitcoind_opts = get_bitcoin_opts()
spv_headers_path = bitcoind_opts['bitcoind_spv_path']
bitcoind_server = bitcoind_opts['bitcoind_server']
rc = SPVClient.sync_header_chain(
spv_headers_path, bitcoind_server, block_id
if not rc:
msg = 'Failed to synchronize SPV header chain up to {}'
return None
# verify block header
rc = SPVClient.block_header_verify(spv_headers_path, block_id, block_hash, block_data)
if not rc:
msg = 'Failed to verify block header for {} against SPV headers'
return None
# verify block txs
rc = SPVClient.block_verify(block_data, block_data['tx'])
if not rc:
msg = 'Failed to verify block transaction IDs for {} against SPV headers'
return None
# sanity check
if tx_index >= len(block_data['tx']):
msg = 'Serial number {} references non-existant transaction {} (out of {} txs)'
log.error(msg.format(serial_number, tx_index, len(block_data['tx'])))
return None
# obtain transaction
txid = block_data['tx'][tx_index]
tx = bitcoind_proxy.getrawtransaction(txid, 1)
# verify tx
rc = SPVClient.tx_verify(block_data['tx'], tx)
if not rc:
msg = 'Failed to verify block transaction {} against SPV headers'
return None
# verify tx index
if tx_index != SPVClient.tx_index(block_data['tx'], tx):
msg = (
'TX index mismatch: serial number identifies '
'transaction number {} ({}), but got transaction {}'
tx_index, block_data['tx'][tx_index],
block_data['tx'][SPVClient.tx_index(block_data['tx'], tx)]
return None
# success!
return tx
def parse_tx_op_return(tx):
Given a transaction, locate its OP_RETURN and parse
out its opcode and payload.
Return (opcode, payload) on success
Return (None, None) if there is no OP_RETURN, or if it's not a blockchain ID operation.
# find OP_RETURN output
op_return = None
outputs = tx['vout']
for out in outputs:
script_key = out['scriptPubKey']['hex']
if int(script_key[0:2], 16) == virtualchain.OPCODE_VALUES['OP_RETURN']:
op_return = script_key.decode('hex')
if op_return is None:
msg = 'transaction has no OP_RETURN output'
log.debug('{}:\n{}'.format(msg, simplejson.dumps(tx)))
return None, None
# [0] is OP_RETURN, [1] is the length; [2:4] are 'id', [4] is opcode
magic = op_return[2:4]
if magic != blockstack_magic_bytes():
# not a blockchain ID operation
msg = 'OP_RETURN output does not encode a blockchain ID operation'
return None, None
opcode, payload = op_return[4], op_return[5:]
return (opcode, payload)
def get_consensus_hash_from_tx(tx):
Given an SPV-verified transaction, extract its consensus hash.
Only works of the tx encodes a NAME_PREORDER, NAMESPACE_PREORDER,
Return hex-encoded consensus hash on success.
Return None on error.
opcode, payload = parse_tx_op_return(tx)
if opcode is None or payload is None:
return None
consensus_hash = payload[-16:].encode('hex')
return consensus_hash
msg = (
'Blockchain ID transaction is not a '
return None
def snv_get_blockstack_ops_at(current_block_id, current_consensus_hash, block_id, consensus_hash, proxy=None):
Simple name verification (snv) lookup:
Use a known-good "current" consensus hash and block ID to
look up a set of name operations from the past, given the previous
point in time's untrusted block ID and consensus hash.
import blockstack
log.debug('verify {}-{} to {}-{}'.format(
current_block_id, current_consensus_hash, block_id, consensus_hash
proxy = get_default_proxy() if proxy is None else proxy
# work backwards in time, using a Merkle skip-list constructed
# by blockstackd over the set of consensus hashes.
next_block_id = current_block_id
prev_blockstack_ops_hashes = {}
prev_consensus_hashes = {
next_block_id: current_consensus_hash
# print 'next_block_id = {}, block_id = {}'.format(next_block_id, block_id)
while next_block_id >= block_id:
# get blockstack_ops_at[ next_block_id ], and all consensus_hash[ next_block_id - 2^i ]
# such that block_id - 2*i > block_id (start at i = 1)
i = 0
blockstack_ops_hash = None
if next_block_id in prev_blockstack_ops_hashes:
blockstack_ops_hash = prev_blockstack_ops_hashes[next_block_id]
blockstack_ops_resp = get_blockstack_ops_hash_at(next_block_id, proxy=proxy)
if 'error' in blockstack_ops_resp:
log.error('get_blockstack_ops_hash_at: {}'.format(blockstack_ops_resp['error']))
return {'error': 'Failed to get nameops: {}'.format(blockstack_ops_resp['error'])}
blockstack_ops_hash = str(blockstack_ops_resp)
prev_blockstack_ops_hashes[next_block_id] = blockstack_ops_hash
log.debug('nameops hash at {}: {}'.format(next_block_id, blockstack_ops_hash))
# find out which consensus hashes we'll need
to_fetch = []
ch_block_ids = []
while next_block_id - (2 ** (i + 1) - 1) >= FIRST_BLOCK_MAINNET:
i += 1
prev_block_id = next_block_id - (2 ** i - 1)
if prev_block_id not in prev_consensus_hashes:
# get the consensus hashes
chs = {}
if to_fetch:
chs = get_consensus_hashes(to_fetch, proxy=proxy)
if 'error' in chs:
msg = 'Failed to get consensus hashes for {}: {}'
log.error(msg.format(to_fetch, chs['error']))
return {'error': 'Failed to get consensus hashes'}
prev_consensus_block_ids = []
for b in ch_block_ids:
# NOTE: we process to_fetch *in decreasing order* so we know when we're missing data
if b not in chs and b not in prev_consensus_hashes:
msg = 'Missing consensus hash response for {} (chs={}, prev_chs={})'
log.error(msg.format(b, chs, prev_consensus_hashes))
return {'error': 'Server did not reply valid data'}
if b in prev_consensus_hashes:
# already got this one
ch = chs[b]
if ch is not None:
prev_consensus_hashes[b] = str(ch)
# no consensus hash for this block and all future blocks
# prev_consensus_hashes_list = [ prev_consensus_hashes[b] for b in ch_block_ids ]
prev_consensus_hashes_list = [
prev_consensus_hashes[b] for b in prev_consensus_block_ids
# calculate the snapshot, and see if it matches
ch = virtualchain.StateEngine.make_snapshot_from_ops_hash(
blockstack_ops_hash, prev_consensus_hashes_list
expected_ch = prev_consensus_hashes[next_block_id]
if ch != expected_ch:
msg = 'Consensus hash mismatch at {}: expected {}, got {} (from {}, {})'
log.error(msg.format(next_block_id, expected_ch, ch, blockstack_ops_hash, prev_consensus_hashes))
return {'error': 'Consensus hash mismatch'}
# advance!
# find the smallest known consensus hash whose block is greater than block_id
current_candidate = next_block_id
found_any = False
for candidate_block_id in prev_consensus_hashes:
if candidate_block_id < block_id:
if candidate_block_id < current_candidate:
current_candidate = candidate_block_id
found_any = True
if not found_any:
next_block_id = current_candidate
# get the final nameops
historic_nameops = get_blockstack_transactions_at(block_id, proxy=proxy)
if isinstance(historic_nameops, dict) and 'error' in historic_nameops:
log.error('Failed to get nameops at {}: {}'.format(block_id, historic_nameops['error']))
return {'error': 'BUG: no nameops found'}
log.debug("nameops at {}\n{}".format(block_id, json.dumps(historic_nameops, sort_keys=True)))
# sanity check...
for historic_op in historic_nameops:
if 'opcode' not in historic_op:
return {'error': 'Invalid/corrupt name operations detected'}
# recover binary op string
if 'op' not in historic_op:
historic_op['op'] = NAME_OPCODES[str(historic_op['opcode'])]
# check integrity
opfields = blockstack.lib.virtualchain_hooks.get_opfields()
serialized_historic_nameops = [
str(op['op'][0]), op, opfields, verbose=True
) for op in historic_nameops
historic_blockstack_ops_hash = virtualchain.StateEngine.make_ops_snapshot(serialized_historic_nameops)
if block_id not in prev_blockstack_ops_hashes:
return {'error': 'Previous block/consensus hash is unreachable from trusted block/consensus hash'}
if historic_blockstack_ops_hash != prev_blockstack_ops_hashes[block_id]:
log.error("historic nameops hash at {}: {}".format(block_id, historic_blockstack_ops_hash))
log.error("prev_blockstack_ops_hashes:\n{}".format(json.dumps(prev_blockstack_ops_hashes, indent=4, sort_keys=True)))
return {
'error': 'Hash mismatch: failed to get operations at {}-{} from {}-{} ({} != {})'.format(
block_id, consensus_hash, current_block_id, current_consensus_hash, historic_blockstack_ops_hash, prev_blockstack_ops_hashes[block_id]
log.debug('{} nameops at {}'.format(len(historic_nameops), block_id))
# strip history
for hn in historic_nameops:
if 'history' in hn.keys():
del hn['history']
return historic_nameops
def snv_name_verify(name, current_block_id, current_consensus_hash, block_id,
consensus_hash, trusted_txid=None, trusted_txindex=None, proxy=None):
Use SNV to verify that a name existed at a particular block ID in the past,
given a later known-good block ID and consensus hash (as well as the previous
untrusted consensus hash)
Return the name's historic nameop(s) on success.
If there are multiple matches, multiple nameops will be returned.
The return value takes the form of {'status': True, 'nameops': [...]}
Return a dict with {'error'} on error
proxy = get_default_proxy() if proxy is None else proxy
historic_nameops = snv_get_blockstack_ops_at(
current_block_id, current_consensus_hash,
block_id, consensus_hash, proxy=proxy
if 'error' in historic_nameops:
return historic_nameops
matching_nameops = []
# find the one we asked for
for nameop in historic_nameops:
# select on more-accurate filters first
if trusted_txindex is not None and nameop['vtxindex'] == trusted_txindex:
matching_nameops = [nameop]
if trusted_txid is not None and nameop['txid'] == trusted_txid:
matching_nameops = [nameop]
if 'name' not in nameop:
if str(nameop['name']) == str(name):
# success!
if matching_nameops:
return {'status': True, 'nameops': matching_nameops}
# not found
log.error('Not found at block {}: "{}"'.format(block_id, name))
return {'error': 'Name not found'}
def snv_lookup(verify_name, verify_block_id,
trusted_serial_number_or_txid_or_consensus_hash, hostport=None, proxy=None, trusted_txid=None):
High-level call to simple name verification:
Given a trusted serial number, txid, or consensus_hash, use it as a trust root to verify that
a previously-registered but untrusted name (@verify_name) exists and was processed
at a given block (@verify_block_id)
Basically, use the trust root to derive a "current" block ID and consensus hash, and
use the untrusted (name, block_id) pair to derive an earlier untrusted block ID and
consensus hash. Then, use the snv_get_blockstack_ops_at() method to verify that the name
existed at the given block ID.
The Blockstack node is not trusted. This algorithm prevents a malicious Blockstack node
from getting the caller to falsely trust @verify_name and @verify_block_id by
using SNV to confirm that:
* the consensus hash at the trust root's block is consistent with @verify_name's
* the consensus hash at @trusted_serial_number's block is consistent with @verify_name's
consensus hash (from @verify_serial_number)
The only way a Blockstack node working with a malicious Sybil can trick the caller is if
both can create a parallel history of name operations such that the final consensus hash
at @trusted_serial_number's block collides. This is necessary, since the client uses
the hash over a block's operations and prior consensus hashes to transitively trust
prior consensus hashes--if the later consensus hash is assumed out-of-band to be valid,
then the transitive closure of all prior consensus hashes will be assumed valid as well.
This means that the only way to drive the valid consensus hash from a prior invalid
consensus hash is to force a hash collision somewhere in the transitive closure, which is infeasible.
NOTE: @trusted_txid is needed for isolating multiple operations in the same name within a single block.
Return the list of nameops in the given verify_block_id that match.
proxy = get_default_proxy(hostport) if proxy is None else proxy
trusted_serial_number_or_txid_or_consensus_hash = str(trusted_serial_number_or_txid_or_consensus_hash)
bitcoind_proxy = get_bitcoind_client()
trusted_serial_number = None
trusted_tx_index = None
trusted_consensus_hash = None
trusted_block_id = None
# what did we get?
hash_len_64 = len(trusted_serial_number_or_txid_or_consensus_hash) == 64
hash_len_32 = len(trusted_serial_number_or_txid_or_consensus_hash) == 32
hash_parts_2 = len(trusted_serial_number_or_txid_or_consensus_hash.split('-')) == 2
hash_is_hex = is_hex(trusted_serial_number_or_txid_or_consensus_hash)
if hash_len_64 and hash_is_hex:
# txid: convert to trusted block ID and consensus hash
trusted_txid = trusted_serial_number_or_txid_or_consensus_hash
trusted_block_hash, trusted_block_data, trusted_tx = txid_to_block_data(trusted_txid, bitcoind_proxy)
if trusted_block_hash is None or trusted_block_data is None or trusted_tx is None:
return {'error': 'Unable to look up given transaction ID'}
# must have a consensus hash
# TOOD: Check why return values are ignored
op, payload = parse_tx_op_return(trusted_tx)
trusted_consensus_hash = get_consensus_hash_from_tx(trusted_tx)
if trusted_consensus_hash is None:
return {'error': 'Tx does not refer to a consensus-bearing transaction'}
# find the block for this consensus hash (it's not the same as the serial number's block ID,
# but that's okay--if the consensus hash in this tx is inauthentic, it will be unreachable
# from the other consensus hash [short of a SHA256 collision])
trusted_block_id = get_block_from_consensus(trusted_consensus_hash, proxy=proxy)
elif hash_len_32 and hash_is_hex:
# consensus hash
trusted_consensus_hash = trusted_serial_number_or_txid_or_consensus_hash
trusted_block_id = get_block_from_consensus(trusted_consensus_hash, proxy=proxy)
if isinstance(trusted_block_id, dict) and 'error' in trusted_block_id:
# got error back
return trusted_block_id
elif hash_parts_2:
# must be a serial number
parts = trusted_serial_number_or_txid_or_consensus_hash.split('-')
trusted_block_id = int(parts[0])
trusted_tx_index = int(parts[1])
log.error('Malformed serial number "{}"'.format(trusted_serial_number_or_txid_or_consensus_hash))
return {'error': 'Did not receive a valid serial number'}
trusted_tx = serial_number_to_tx(trusted_serial_number_or_txid_or_consensus_hash, bitcoind_proxy)
if trusted_tx is None:
return {'error': 'Unable to convert given serial number into transaction'}
# tx must have a consensus hash
# TOOD: Check why return values are ignored
op, payload = parse_tx_op_return(trusted_tx)
trusted_consensus_hash = get_consensus_hash_from_tx(trusted_tx)
if trusted_consensus_hash is None:
return {'error': 'Tx does not refer to a consensus-bearing transaction'}
# find the block for this consensus hash (it's not the same as the serial number's block ID,
# but that's okay--if the consensus hash in this tx is inauthentic, it will be unreachable
# from the other consensus hash [short of a SHA256 collision])
trusted_block_id = get_block_from_consensus(trusted_consensus_hash, proxy=proxy)
if isinstance(trusted_block_id, dict) and 'error' in trusted_block_id:
# got error back
return trusted_block_id
msg = 'Did not receive a valid txid, consensus hash, or serial number ({})'
return {'error': msg.format(trusted_serial_number_or_txid_or_consensus_hash)}
if trusted_block_id < verify_block_id:
msg = 'Trusted block/consensus hash came before the untrusted block/consensus hash'
return {'error': msg}
# go verify the name
verify_consensus_hash = get_consensus_at(verify_block_id, proxy=proxy)
historic_namerecs = snv_name_verify(
verify_name, trusted_block_id, trusted_consensus_hash,
verify_block_id, verify_consensus_hash,
trusted_txid=trusted_txid, trusted_txindex=trusted_tx_index
if 'error' in historic_namerecs:
return historic_namerecs
return historic_namerecs['nameops']
# backwards compatibility
lookup_snv = snv_lookup
You can’t perform that action at this time.