Permalink
Browse files

Merge pull request #6103

  • Loading branch information...
Jeff Garzik
Jeff Garzik committed Sep 16, 2015
2 parents 9733bc9 + 029e278 commit 13b828270a6f912aeb24dee87108c6bda83d1b96
View
@@ -137,6 +137,12 @@ AC_ARG_ENABLE([glibc-back-compat],
[use_glibc_compat=$enableval],
[use_glibc_compat=no])
+AC_ARG_ENABLE([zmq],
+ [AC_HELP_STRING([--disable-zmq],
+ [Disable ZMQ notifications])],
+ [use_zmq=$enableval],
+ [use_zmq=yes])
+
AC_ARG_WITH([protoc-bindir],[AS_HELP_STRING([--with-protoc-bindir=BIN_DIR],[specify protoc bin path])], [protoc_bin_path=$withval], [])
# Enable debug
@@ -833,6 +839,22 @@ if test x$bitcoin_enable_qt != xno; then
fi
fi
+# conditional search for and use libzmq
+AC_MSG_CHECKING([whether to build ZMQ support])
+if test "x$use_zmq" = "xyes"; then
+ AC_MSG_RESULT([yes])
+ PKG_CHECK_MODULES([ZMQ],[libzmq],
+ [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])],
+ [AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])
+ AC_MSG_WARN([libzmq not found, disabling])
+ use_zmq=no])
+else
+ AC_MSG_RESULT([no, --disable-zmq used])
+ AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])
+fi
+
+AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"])
+
AC_MSG_CHECKING([whether to build test_bitcoin])
if test x$use_tests = xyes; then
AC_MSG_RESULT([yes])
View
@@ -0,0 +1,37 @@
+#!/usr/bin/env python2
+
+import array
+import binascii
+import zmq
+
+port = 28332
+
+zmqContext = zmq.Context()
+zmqSubSocket = zmqContext.socket(zmq.SUB)
+zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashblock")
+zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashtx")
+zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "rawblock")
+zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "rawtx")
+zmqSubSocket.connect("tcp://127.0.0.1:%i" % port)
+
+try:
+ while True:
+ msg = zmqSubSocket.recv_multipart()
+ topic = str(msg[0])
+ body = msg[1]
+
+ if topic == "hashblock":
+ print "- HASH BLOCK -"
+ print binascii.hexlify(body)
+ elif topic == "hashtx":
+ print '- HASH TX -'
+ print binascii.hexlify(body)
+ elif topic == "rawblock":
+ print "- RAW BLOCK HEADER -"
+ print binascii.hexlify(body[:80])
+ elif topic == "rawtx":
+ print '- RAW TX -'
+ print binascii.hexlify(body)
+
+except KeyboardInterrupt:
+ zmqContext.destroy()
@@ -1,4 +1,6 @@
packages:=boost openssl libevent
+packages_darwin:=zeromq
+packages_linux:=zeromq
native_packages := native_ccache native_comparisontool
qt_native_packages = native_protobuf
View
@@ -0,0 +1,26 @@
+package=zeromq
+$(package)_version=4.0.4
+$(package)_download_path=http://download.zeromq.org
+$(package)_file_name=$(package)-$($(package)_version).tar.gz
+$(package)_sha256_hash=1ef71d46e94f33e27dd5a1661ed626cd39be4d2d6967792a275040e34457d399
+
+define $(package)_set_vars
+ $(package)_config_opts=--without-documentation --disable-shared
+ $(package)_config_opts_linux=--with-pic
+endef
+
+define $(package)_config_cmds
+ $($(package)_autoconf)
+endef
+
+define $(package)_build_cmds
+ $(MAKE) -C src
+endef
+
+define $(package)_stage_cmds
+ $(MAKE) -C src DESTDIR=$($(package)_staging_dir) install
+endef
+
+define $(package)_postprocess_cmds
+ rm -rf bin share
+endef
View
@@ -0,0 +1,98 @@
+# Block and Transaction Broadcasting With ZeroMQ
+
+[ZeroMQ](http://zeromq.org/) is a lightweight wrapper around TCP
+connections, inter-process communications, and shared-memory,
+providing various message-oriented semantics such as publish/subcribe,
+request/reply, and push/pull.
+
+The Bitcoin Core daemon can be configured to act as a trusted "border
+router", implementing the bitcoin wire protocol and relay, making
+consensus decisions, maintaining the local blockchain database,
+broadcasting locally generated transactions into the network, and
+providing a queryable RPC interface to interact on a polled basis for
+requesting blockchain related data. However, there exists only a
+limited service to notify external software of events like the arrival
+of new blocks or transactions.
+
+The ZeroMQ facility implements a notification interface through a
+set of specific notifiers. Currently there are notifiers that publish
+blocks and transactions. This read-only facility requires only the
+connection of a corresponding ZeroMQ subscriber port in receiving
+software; it is not authenticated nor is there any two-way protocol
+involvement. Therefore, subscribers should validate the received data
+since it may be out of date, incomplete or even invalid.
+
+ZeroMQ sockets are self-connecting and self-healing; that is, connects
+made between two endpoints will be automatically restored after an
+outage, and either end may be freely started or stopped in any order.
+
+Because ZeroMQ is message oriented, subscribers receive transactions
+and blocks all-at-once and do not need to implement any sort of
+buffering or reassembly.
+
+## Prerequisites
+
+The ZeroMQ feature in Bitcoin Core uses only a very small part of the
+ZeroMQ C API, and is thus compatible with any version of ZeroMQ
+from 2.1 onward, including all versions in the 3.x and 4.x release
+series. Typically, it is packaged by distributions as something like
+*libzmq-dev*.
+
+The C++ wrapper for ZeroMQ is *not* needed.
+
+## Enabling
+
+By default, the ZeroMQ port functionality is enabled. Two steps are
+required to enable--compiling in the ZeroMQ code, and configuring
+runtime operation on the command-line or configuration file.
+
+ $ ./configure --enable-zmq (other options)
+
+This will produce a binary that is capable of providing the ZeroMQ
+facility, but will not do so until also configured properly.
+
+## Usage
+
+Currently, the following notifications are supported:
+
+ -zmqpubhashtx=address
+ -zmqpubhashblock=address
+ -zmqpubrawblock=address
+ -zmqpubrawtx=address
+
+The socket type is PUB and the address must be a valid ZeroMQ
+socket address. The same address can be used in more than one notification.
+
+For instance:
+
+ $ bitcoind -zmqpubhashtx=tcp://127.0.0.1:28332 -zmqpubrawtx=ipc:///tmp/bitcoind.tx.raw
+
+Each PUB notification has a topic and body, where the header
+corresponds to the notification type. For instance, for the notification
+`-zmqpubhashtx` the topic is `hashtx` (no null terminator) and the body is the
+hexadecimal transaction hash (32 bytes).
+
+These options can also be provided in bitcoin.conf.
+
+ZeroMQ endpoint specifiers for TCP (and others) are documented in the
+[ZeroMQ API](http://api.zeromq.org).
+
+Client side, then, the ZeroMQ subscriber socket must have the
+ZMQ_SUBSCRIBE option set to one or either of these prefixes (for instance, just `hash`); without
+doing so will result in no messages arriving. Please see `contrib/zmq/zmq_sub.py`
+for a working example.
+
+## Remarks
+
+From the perspective of bitcoind, the ZeroMQ socket is write-only; PUB
+sockets don't even have a read function. Thus, there is no state
+introduced into bitcoind directly. Furthermore, no information is
+broadcast that wasn't already received from the public P2P network.
+
+No authentication or authorization is done on connecting clients; it
+is assumed that the ZeroMQ port is exposed only to trusted entities,
+using other means such as firewalling.
+
+Note that when the block chain tip changes, a reorganisation may occur and just
+the tip will be notified. It is up to the subscriber to retrieve the chain
+from the last known block to the new tip.
@@ -59,6 +59,10 @@ testScriptsExt=(
'p2p-acceptblock.py'
);
+if [ "x$ENABLE_ZMQ" = "x1" ]; then
+ testScripts=( ${testScripts[@]} 'zmq_test.py' )
+fi
+
extArg="-extended"
passOn=${@#$extArg}
@@ -10,6 +10,7 @@ EXEEXT="@EXEEXT@"
@ENABLE_WALLET_TRUE@ENABLE_WALLET=1
@BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=1
@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=1
+@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=1
REAL_BITCOIND="$BUILDDIR/src/bitcoind${EXEEXT}"
REAL_BITCOINCLI="$BUILDDIR/src/bitcoin-cli${EXEEXT}"
View
@@ -0,0 +1,93 @@
+#!/usr/bin/env python2
+# Copyright (c) 2015 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# Test ZMQ interface
+#
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import *
+import zmq
+import binascii
+from test_framework.mininode import hash256
+
+try:
+ import http.client as httplib
+except ImportError:
+ import httplib
+try:
+ import urllib.parse as urlparse
+except ImportError:
+ import urlparse
+
+class ZMQTest (BitcoinTestFramework):
+
+ port = 28332
+
+ def setup_nodes(self):
+ self.zmqContext = zmq.Context()
+ self.zmqSubSocket = self.zmqContext.socket(zmq.SUB)
+ self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashblock")
+ self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashtx")
+ self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % self.port)
+ # Note: proxies are not used to connect to local nodes
+ # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost
+ return start_nodes(4, self.options.tmpdir, extra_args=[
+ ['-zmqpubhashtx=tcp://127.0.0.1:'+str(self.port), '-zmqpubhashblock=tcp://127.0.0.1:'+str(self.port)],
+ [],
+ [],
+ []
+ ])
+
+ def run_test(self):
+ self.sync_all()
+
+ genhashes = self.nodes[0].generate(1);
+ self.sync_all()
+
+ print "listen..."
+ msg = self.zmqSubSocket.recv_multipart()
+ topic = str(msg[0])
+ body = msg[1]
+
+ msg = self.zmqSubSocket.recv_multipart()
+ topic = str(msg[0])
+ body = msg[1]
+ blkhash = binascii.hexlify(body)
+
+ assert_equal(genhashes[0], blkhash) #blockhash from generate must be equal to the hash received over zmq
+
+ n = 10
+ genhashes = self.nodes[1].generate(n);
+ self.sync_all()
+
+ zmqHashes = []
+ for x in range(0,n*2):
+ msg = self.zmqSubSocket.recv_multipart()
+ topic = str(msg[0])
+ body = msg[1]
+ if topic == "hashblock":
+ zmqHashes.append(binascii.hexlify(body))
+
+ for x in range(0,n):
+ assert_equal(genhashes[x], zmqHashes[x]) #blockhash from generate must be equal to the hash received over zmq
+
+ #test tx from a second node
+ hashRPC = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0)
+ self.sync_all()
+
+ #now we should receive a zmq msg because the tx was broadcastet
+ msg = self.zmqSubSocket.recv_multipart()
+ topic = str(msg[0])
+ body = msg[1]
+ hashZMQ = ""
+ if topic == "hashtx":
+ hashZMQ = binascii.hexlify(body)
+
+ assert_equal(hashRPC, hashZMQ) #blockhash from generate must be equal to the hash received over zmq
+
+
+if __name__ == '__main__':
+ ZMQTest ().main ()
View
@@ -48,6 +48,9 @@ if ENABLE_WALLET
BITCOIN_INCLUDES += $(BDB_CPPFLAGS)
EXTRA_LIBRARIES += libbitcoin_wallet.a
endif
+if ENABLE_ZMQ
+EXTRA_LIBRARIES += libbitcoin_zmq.a
+endif
if BUILD_BITCOIN_LIBS
lib_LTLIBRARIES = libbitcoinconsensus.la
@@ -157,7 +160,12 @@ BITCOIN_CORE_H = \
wallet/db.h \
wallet/wallet.h \
wallet/wallet_ismine.h \
- wallet/walletdb.h
+ wallet/walletdb.h \
+ zmq/zmqabstractnotifier.h \
+ zmq/zmqconfig.h\
+ zmq/zmqnotificationinterface.h \
+ zmq/zmqpublishnotifier.h
+
obj/build.h: FORCE
@$(MKDIR_P) $(builddir)/obj
@@ -199,6 +207,17 @@ libbitcoin_server_a_SOURCES = \
validationinterface.cpp \
$(BITCOIN_CORE_H)
+if ENABLE_ZMQ
+LIBBITCOIN_ZMQ=libbitcoin_zmq.a
+
+libbitcoin_zmq_a_CPPFLAGS = $(BITCOIN_INCLUDES)
+libbitcoin_zmq_a_SOURCES = \
+ zmq/zmqabstractnotifier.cpp \
+ zmq/zmqnotificationinterface.cpp \
+ zmq/zmqpublishnotifier.cpp
+endif
+
+
# wallet: shared between bitcoind and bitcoin-qt, but only linked
# when wallet enabled
libbitcoin_wallet_a_CPPFLAGS = $(BITCOIN_INCLUDES)
@@ -320,12 +339,15 @@ bitcoind_LDADD = \
$(LIBMEMENV) \
$(LIBSECP256K1)
+if ENABLE_ZMQ
+bitcoind_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
+endif
+
if ENABLE_WALLET
bitcoind_LDADD += libbitcoin_wallet.a
endif
bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
-#
# bitcoin-cli binary #
bitcoin_cli_SOURCES = bitcoin-cli.cpp
View
@@ -361,6 +361,9 @@ qt_bitcoin_qt_LDADD = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER)
if ENABLE_WALLET
qt_bitcoin_qt_LDADD += $(LIBBITCOIN_WALLET)
endif
+if ENABLE_ZMQ
+qt_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
+endif
qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_UNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \
$(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
$(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
@@ -30,6 +30,9 @@ qt_test_test_bitcoin_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_SERVER)
if ENABLE_WALLET
qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_WALLET)
endif
+if ENABLE_ZMQ
+qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
+endif
qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_UNIVALUE) $(LIBLEVELDB) \
$(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
$(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
Oops, something went wrong.

0 comments on commit 13b8282

Please sign in to comment.