Skip to content
Permalink
Browse files

Merge pull request #6103

  • Loading branch information
Jeff Garzik
Jeff Garzik committed Sep 16, 2015
2 parents 9733bc9 + 029e278 commit 13b828270a6f912aeb24dee87108c6bda83d1b96
@@ -137,6 +137,12 @@ AC_ARG_ENABLE([glibc-back-compat],
[use_glibc_compat=$enableval],
[use_glibc_compat=no])

AC_ARG_ENABLE([zmq],
[AC_HELP_STRING([--disable-zmq],
[Disable ZMQ notifications])],
[use_zmq=$enableval],
[use_zmq=yes])

AC_ARG_WITH([protoc-bindir],[AS_HELP_STRING([--with-protoc-bindir=BIN_DIR],[specify protoc bin path])], [protoc_bin_path=$withval], [])

# Enable debug
@@ -833,6 +839,22 @@ if test x$bitcoin_enable_qt != xno; then
fi
fi

# conditional search for and use libzmq
AC_MSG_CHECKING([whether to build ZMQ support])
if test "x$use_zmq" = "xyes"; then
AC_MSG_RESULT([yes])
PKG_CHECK_MODULES([ZMQ],[libzmq],
[AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])],
[AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])
AC_MSG_WARN([libzmq not found, disabling])
use_zmq=no])
else
AC_MSG_RESULT([no, --disable-zmq used])
AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])
fi

AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"])

AC_MSG_CHECKING([whether to build test_bitcoin])
if test x$use_tests = xyes; then
AC_MSG_RESULT([yes])
@@ -0,0 +1,37 @@
#!/usr/bin/env python2

import array
import binascii
import zmq

port = 28332

zmqContext = zmq.Context()
zmqSubSocket = zmqContext.socket(zmq.SUB)
zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashblock")
zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashtx")
zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "rawblock")
zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "rawtx")
zmqSubSocket.connect("tcp://127.0.0.1:%i" % port)

try:
while True:
msg = zmqSubSocket.recv_multipart()
topic = str(msg[0])
body = msg[1]

if topic == "hashblock":
print "- HASH BLOCK -"
print binascii.hexlify(body)
elif topic == "hashtx":
print '- HASH TX -'
print binascii.hexlify(body)
elif topic == "rawblock":
print "- RAW BLOCK HEADER -"
print binascii.hexlify(body[:80])
elif topic == "rawtx":
print '- RAW TX -'
print binascii.hexlify(body)

except KeyboardInterrupt:
zmqContext.destroy()
@@ -1,4 +1,6 @@
packages:=boost openssl libevent
packages_darwin:=zeromq
packages_linux:=zeromq
native_packages := native_ccache native_comparisontool

qt_native_packages = native_protobuf
@@ -0,0 +1,26 @@
package=zeromq
$(package)_version=4.0.4
$(package)_download_path=http://download.zeromq.org
$(package)_file_name=$(package)-$($(package)_version).tar.gz
$(package)_sha256_hash=1ef71d46e94f33e27dd5a1661ed626cd39be4d2d6967792a275040e34457d399

define $(package)_set_vars
$(package)_config_opts=--without-documentation --disable-shared
$(package)_config_opts_linux=--with-pic
endef

define $(package)_config_cmds
$($(package)_autoconf)
endef

define $(package)_build_cmds
$(MAKE) -C src
endef

define $(package)_stage_cmds
$(MAKE) -C src DESTDIR=$($(package)_staging_dir) install
endef

define $(package)_postprocess_cmds
rm -rf bin share
endef
@@ -0,0 +1,98 @@
# Block and Transaction Broadcasting With ZeroMQ

[ZeroMQ](http://zeromq.org/) is a lightweight wrapper around TCP
connections, inter-process communications, and shared-memory,
providing various message-oriented semantics such as publish/subcribe,
request/reply, and push/pull.

The Bitcoin Core daemon can be configured to act as a trusted "border
router", implementing the bitcoin wire protocol and relay, making
consensus decisions, maintaining the local blockchain database,
broadcasting locally generated transactions into the network, and
providing a queryable RPC interface to interact on a polled basis for
requesting blockchain related data. However, there exists only a
limited service to notify external software of events like the arrival
of new blocks or transactions.

The ZeroMQ facility implements a notification interface through a
set of specific notifiers. Currently there are notifiers that publish
blocks and transactions. This read-only facility requires only the
connection of a corresponding ZeroMQ subscriber port in receiving
software; it is not authenticated nor is there any two-way protocol
involvement. Therefore, subscribers should validate the received data
since it may be out of date, incomplete or even invalid.

ZeroMQ sockets are self-connecting and self-healing; that is, connects
made between two endpoints will be automatically restored after an
outage, and either end may be freely started or stopped in any order.

Because ZeroMQ is message oriented, subscribers receive transactions
and blocks all-at-once and do not need to implement any sort of
buffering or reassembly.

## Prerequisites

The ZeroMQ feature in Bitcoin Core uses only a very small part of the
ZeroMQ C API, and is thus compatible with any version of ZeroMQ
from 2.1 onward, including all versions in the 3.x and 4.x release
series. Typically, it is packaged by distributions as something like
*libzmq-dev*.

The C++ wrapper for ZeroMQ is *not* needed.

## Enabling

By default, the ZeroMQ port functionality is enabled. Two steps are
required to enable--compiling in the ZeroMQ code, and configuring
runtime operation on the command-line or configuration file.

$ ./configure --enable-zmq (other options)

This will produce a binary that is capable of providing the ZeroMQ
facility, but will not do so until also configured properly.

## Usage

Currently, the following notifications are supported:

-zmqpubhashtx=address
-zmqpubhashblock=address
-zmqpubrawblock=address
-zmqpubrawtx=address

The socket type is PUB and the address must be a valid ZeroMQ
socket address. The same address can be used in more than one notification.

For instance:

$ bitcoind -zmqpubhashtx=tcp://127.0.0.1:28332 -zmqpubrawtx=ipc:///tmp/bitcoind.tx.raw

Each PUB notification has a topic and body, where the header
corresponds to the notification type. For instance, for the notification
`-zmqpubhashtx` the topic is `hashtx` (no null terminator) and the body is the
hexadecimal transaction hash (32 bytes).

These options can also be provided in bitcoin.conf.

ZeroMQ endpoint specifiers for TCP (and others) are documented in the
[ZeroMQ API](http://api.zeromq.org).

Client side, then, the ZeroMQ subscriber socket must have the
ZMQ_SUBSCRIBE option set to one or either of these prefixes (for instance, just `hash`); without
doing so will result in no messages arriving. Please see `contrib/zmq/zmq_sub.py`
for a working example.

## Remarks

From the perspective of bitcoind, the ZeroMQ socket is write-only; PUB
sockets don't even have a read function. Thus, there is no state
introduced into bitcoind directly. Furthermore, no information is
broadcast that wasn't already received from the public P2P network.

No authentication or authorization is done on connecting clients; it
is assumed that the ZeroMQ port is exposed only to trusted entities,
using other means such as firewalling.

Note that when the block chain tip changes, a reorganisation may occur and just
the tip will be notified. It is up to the subscriber to retrieve the chain
from the last known block to the new tip.
@@ -59,6 +59,10 @@ testScriptsExt=(
'p2p-acceptblock.py'
);

if [ "x$ENABLE_ZMQ" = "x1" ]; then
testScripts=( ${testScripts[@]} 'zmq_test.py' )
fi

extArg="-extended"
passOn=${@#$extArg}

@@ -10,6 +10,7 @@ EXEEXT="@EXEEXT@"
@ENABLE_WALLET_TRUE@ENABLE_WALLET=1
@BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=1
@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=1
@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=1

REAL_BITCOIND="$BUILDDIR/src/bitcoind${EXEEXT}"
REAL_BITCOINCLI="$BUILDDIR/src/bitcoin-cli${EXEEXT}"
@@ -0,0 +1,93 @@
#!/usr/bin/env python2
# Copyright (c) 2015 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

#
# Test ZMQ interface
#

from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *
import zmq
import binascii
from test_framework.mininode import hash256

try:
import http.client as httplib
except ImportError:
import httplib
try:
import urllib.parse as urlparse
except ImportError:
import urlparse

class ZMQTest (BitcoinTestFramework):

port = 28332

def setup_nodes(self):
self.zmqContext = zmq.Context()
self.zmqSubSocket = self.zmqContext.socket(zmq.SUB)
self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashblock")
self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashtx")
self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % self.port)
# Note: proxies are not used to connect to local nodes
# this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost
return start_nodes(4, self.options.tmpdir, extra_args=[
['-zmqpubhashtx=tcp://127.0.0.1:'+str(self.port), '-zmqpubhashblock=tcp://127.0.0.1:'+str(self.port)],
[],
[],
[]
])

def run_test(self):
self.sync_all()

genhashes = self.nodes[0].generate(1);
self.sync_all()

print "listen..."
msg = self.zmqSubSocket.recv_multipart()
topic = str(msg[0])
body = msg[1]

msg = self.zmqSubSocket.recv_multipart()
topic = str(msg[0])
body = msg[1]
blkhash = binascii.hexlify(body)

assert_equal(genhashes[0], blkhash) #blockhash from generate must be equal to the hash received over zmq

n = 10
genhashes = self.nodes[1].generate(n);
self.sync_all()

zmqHashes = []
for x in range(0,n*2):
msg = self.zmqSubSocket.recv_multipart()
topic = str(msg[0])
body = msg[1]
if topic == "hashblock":
zmqHashes.append(binascii.hexlify(body))

for x in range(0,n):
assert_equal(genhashes[x], zmqHashes[x]) #blockhash from generate must be equal to the hash received over zmq

#test tx from a second node
hashRPC = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0)
self.sync_all()

#now we should receive a zmq msg because the tx was broadcastet
msg = self.zmqSubSocket.recv_multipart()
topic = str(msg[0])
body = msg[1]
hashZMQ = ""
if topic == "hashtx":
hashZMQ = binascii.hexlify(body)

assert_equal(hashRPC, hashZMQ) #blockhash from generate must be equal to the hash received over zmq


if __name__ == '__main__':
ZMQTest ().main ()
@@ -48,6 +48,9 @@ if ENABLE_WALLET
BITCOIN_INCLUDES += $(BDB_CPPFLAGS)
EXTRA_LIBRARIES += libbitcoin_wallet.a
endif
if ENABLE_ZMQ
EXTRA_LIBRARIES += libbitcoin_zmq.a
endif

if BUILD_BITCOIN_LIBS
lib_LTLIBRARIES = libbitcoinconsensus.la
@@ -157,7 +160,12 @@ BITCOIN_CORE_H = \
wallet/db.h \
wallet/wallet.h \
wallet/wallet_ismine.h \
wallet/walletdb.h
wallet/walletdb.h \
zmq/zmqabstractnotifier.h \
zmq/zmqconfig.h\
zmq/zmqnotificationinterface.h \
zmq/zmqpublishnotifier.h


obj/build.h: FORCE
@$(MKDIR_P) $(builddir)/obj
@@ -199,6 +207,17 @@ libbitcoin_server_a_SOURCES = \
validationinterface.cpp \
$(BITCOIN_CORE_H)

if ENABLE_ZMQ
LIBBITCOIN_ZMQ=libbitcoin_zmq.a

libbitcoin_zmq_a_CPPFLAGS = $(BITCOIN_INCLUDES)
libbitcoin_zmq_a_SOURCES = \
zmq/zmqabstractnotifier.cpp \
zmq/zmqnotificationinterface.cpp \
zmq/zmqpublishnotifier.cpp
endif


# wallet: shared between bitcoind and bitcoin-qt, but only linked
# when wallet enabled
libbitcoin_wallet_a_CPPFLAGS = $(BITCOIN_INCLUDES)
@@ -320,12 +339,15 @@ bitcoind_LDADD = \
$(LIBMEMENV) \
$(LIBSECP256K1)

if ENABLE_ZMQ
bitcoind_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
endif

if ENABLE_WALLET
bitcoind_LDADD += libbitcoin_wallet.a
endif

bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
#

# bitcoin-cli binary #
bitcoin_cli_SOURCES = bitcoin-cli.cpp
@@ -361,6 +361,9 @@ qt_bitcoin_qt_LDADD = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER)
if ENABLE_WALLET
qt_bitcoin_qt_LDADD += $(LIBBITCOIN_WALLET)
endif
if ENABLE_ZMQ
qt_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
endif
qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_UNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \
$(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
$(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
@@ -30,6 +30,9 @@ qt_test_test_bitcoin_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_SERVER)
if ENABLE_WALLET
qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_WALLET)
endif
if ENABLE_ZMQ
qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
endif
qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_UNIVALUE) $(LIBLEVELDB) \
$(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
$(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \

0 comments on commit 13b8282

Please sign in to comment.
You can’t perform that action at this time.