getblocktemplate: longpolling support (v2)

qa/rpc-tests/getblocktemplate.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# Copyright (c) 2014 The Bitcoin Core developers
+# Distributed under the MIT/X11 software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# Exercise the listtransactions API
+
+from test_framework import BitcoinTestFramework
+from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
+from util import *
+
+
+def check_array_result(object_array, to_match, expected):
+    """
+    Pass in array of JSON objects, a dictionary with key/value pairs
+    to match against, and another dictionary with expected key/value
+    pairs.
+    """
+    num_matched = 0
+    for item in object_array:
+        all_match = True
+        for key,value in to_match.items():
+            if item[key] != value:
+                all_match = False
+        if not all_match:
+            continue
+        for key,value in expected.items():
+            if item[key] != value:
+                raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value)))
+            num_matched = num_matched+1
+    if num_matched == 0:
+        raise AssertionError("No objects matched %s"%(str(to_match)))
+
+import threading
+
+class LongpollThread(threading.Thread):
+    def __init__(self, node):
+        threading.Thread.__init__(self)
+        # query current longpollid
+        templat = node.getblocktemplate()
+        self.longpollid = templat['longpollid']
+        # create a new connection to the node, we can't use the same
+        # connection from two threads
+        self.node = AuthServiceProxy(node.url, timeout=600)
+
+    def run(self):
+        self.node.getblocktemplate({'longpollid':self.longpollid})
+
+class GetBlockTemplateTest(BitcoinTestFramework):
+    '''
+    Test longpolling with getblocktemplate.
+    '''
+
+    def run_test(self, nodes):
+        print "Warning: this test will take about 70 seconds in the best case. Be patient."
+        nodes[0].setgenerate(True, 10)
+        templat = nodes[0].getblocktemplate()
+        longpollid = templat['longpollid']
+        # longpollid should not change between successive invocations if nothing else happens
+        templat2 = nodes[0].getblocktemplate()
+        assert(templat2['longpollid'] == longpollid)
+
+        # Test 1: test that the longpolling wait if we do nothing
+        thr = LongpollThread(nodes[0])
+        thr.start()
+        # check that thread still lives
+        thr.join(5)  # wait 5 seconds or until thread exits
+        assert(thr.is_alive())
+
+        # Test 2: test that longpoll will terminate if another node generates a block
+        nodes[1].setgenerate(True, 1)  # generate a block on another node
+        # check that thread will exit now that new transaction entered mempool
+        thr.join(5)  # wait 5 seconds or until thread exits
+        assert(not thr.is_alive())
+
+        # Test 3: test that longpoll will terminate if we generate a block ourselves
+        thr = LongpollThread(nodes[0])
+        thr.start()
+        nodes[0].setgenerate(True, 1)  # generate a block on another node
+        thr.join(5)  # wait 5 seconds or until thread exits
+        assert(not thr.is_alive())
+
+        # Test 4: test that introducing a new transaction into the mempool will terminate the longpoll
+        thr = LongpollThread(nodes[0])
+        thr.start()
+        # generate a random transaction and submit it
+        (txid, txhex, fee) = random_transaction(nodes, Decimal("1.1"), Decimal("0.0"), Decimal("0.001"), 20)
+        # after one minute, every 10 seconds the mempool is probed, so in 80 seconds it should have returned
+        thr.join(60 + 20)
+        assert(not thr.is_alive())
+
+if __name__ == '__main__':
+    GetBlockTemplateTest().main()
+

qa/rpc-tests/util.py

@@ -156,7 +156,9 @@ def start_node(i, dir, extra_args=None, rpchost=None):
                           ["-rpcwait", "getblockcount"], stdout=devnull)
     devnull.close()
     url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i))
-    return AuthServiceProxy(url)
+    proxy = AuthServiceProxy(url)
+    proxy.url = url # store URL on proxy for info
+    return proxy
 
 def start_nodes(num_nodes, dir, extra_args=None, rpchost=None):
     """

src/main.cpp

@@ -41,6 +41,8 @@ CCriticalSection cs_main;
 map<uint256, CBlockIndex*> mapBlockIndex;
 CChain chainActive;
 int64_t nTimeBestReceived = 0;
+CWaitableCriticalSection csBestBlock;
+CConditionVariable cvBlockChange;
 int nScriptCheckThreads = 0;
 bool fImporting = false;
 bool fReindex = false;
@@ -1944,11 +1946,14 @@ void static UpdateTip(CBlockIndex *pindexNew) {
     // New best block
     nTimeBestReceived = GetTime();
     mempool.AddTransactionsUpdated(1);
+
     LogPrintf("UpdateTip: new best=%s  height=%d  log2_work=%.8g  tx=%lu  date=%s progress=%f\n",
       chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx,
       DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()),
       Checkpoints::GuessVerificationProgress(chainActive.Tip()));
 
+    cvBlockChange.notify_all();
+
     // Check the version of the last 100 blocks to see if we need to upgrade:
     if (!fIsInitialDownload)
     {

src/main.h

@@ -87,6 +87,8 @@ extern uint64_t nLastBlockTx;
 extern uint64_t nLastBlockSize;
 extern const std::string strMessageMagic;
 extern int64_t nTimeBestReceived;
+extern CWaitableCriticalSection csBestBlock;
+extern CConditionVariable cvBlockChange;
 extern bool fImporting;
 extern bool fReindex;
 extern bool fBenchmark;

src/rpcmining.cpp

@@ -324,6 +324,7 @@ Value getblocktemplate(const Array& params, bool fHelp)
          );
 
     std::string strMode = "template";
+    Value lpval = Value::null;
     if (params.size() > 0)
     {
         const Object& oparam = params[0].get_obj();
@@ -336,6 +337,7 @@ Value getblocktemplate(const Array& params, bool fHelp)
         }
         else
             throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode");
+        lpval = find_value(oparam, "longpollid");
     }
 
     if (strMode != "template")
@@ -347,8 +349,63 @@ Value getblocktemplate(const Array& params, bool fHelp)
     if (IsInitialBlockDownload())
         throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Bitcoin is downloading blocks...");
 
-    // Update block
     static unsigned int nTransactionsUpdatedLast;
+
+    if (lpval.type() != null_type)
+    {
+        // Wait to respond until either the best block changes, OR a minute has passed and there are more transactions
+        uint256 hashWatchedChain;
+        boost::system_time checktxtime;
+        unsigned int nTransactionsUpdatedLastLP;
+
+        if (lpval.type() == str_type)
+        {
+            // Format: <hashBestChain><nTransactionsUpdatedLast>
+            std::string lpstr = lpval.get_str();
+
+            hashWatchedChain.SetHex(lpstr.substr(0, 64));
+            nTransactionsUpdatedLastLP = atoi64(lpstr.substr(64));
+        }
+        else
+        {
+            // NOTE: Spec does not specify behaviour for non-string longpollid, but this makes testing easier
+            hashWatchedChain = chainActive.Tip()->GetBlockHash();
+            nTransactionsUpdatedLastLP = nTransactionsUpdatedLast;
+        }
+
+        // Release the wallet and main lock while waiting
+#ifdef ENABLE_WALLET
+        if(pwalletMain)
+            LEAVE_CRITICAL_SECTION(pwalletMain->cs_wallet);
+#endif
+        LEAVE_CRITICAL_SECTION(cs_main);
+        {
+            checktxtime = boost::get_system_time() + boost::posix_time::minutes(1);
+
+            boost::unique_lock<boost::mutex> lock(csBestBlock);
+            while (chainActive.Tip()->GetBlockHash() == hashWatchedChain && IsRPCRunning())
+            {
+                if (!cvBlockChange.timed_wait(lock, checktxtime))
+                {
+                    // Timeout: Check transactions for update
+                    if (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLastLP)
+                        break;
+                    checktxtime += boost::posix_time::seconds(10);
+                }
+            }
+        }
+        ENTER_CRITICAL_SECTION(cs_main);
+#ifdef ENABLE_WALLET
+        if(pwalletMain)
+            ENTER_CRITICAL_SECTION(pwalletMain->cs_wallet);
+#endif
+
+        if (!IsRPCRunning())
+            throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down");
+        // TODO: Maybe recheck connections/IBD and (if something wrong) send an expires-immediately template to stop miners?
+    }
+
+    // Update block
     static CBlockIndex* pindexPrev;
     static int64_t nStart;
     static CBlockTemplate* pblocktemplate;
@@ -436,6 +493,7 @@ Value getblocktemplate(const Array& params, bool fHelp)
     result.push_back(Pair("transactions", transactions));
     result.push_back(Pair("coinbaseaux", aux));
     result.push_back(Pair("coinbasevalue", (int64_t)pblock->vtx[0].vout[0].nValue));
+    result.push_back(Pair("longpollid", chainActive.Tip()->GetBlockHash().GetHex() + i64tostr(nTransactionsUpdatedLast)));
     result.push_back(Pair("target", hashTarget.GetHex()));
     result.push_back(Pair("mintime", (int64_t)pindexPrev->GetMedianTimePast()+1));
     result.push_back(Pair("mutable", aMutable));

src/rpcserver.cpp

@@ -32,6 +32,7 @@ using namespace std;
 
 static std::string strRPCUserColonPass;
 
+static bool fRPCRunning = false;
 // These are created by StartRPCThreads, destroyed in StopRPCThreads
 static asio::io_service* rpc_io_service = NULL;
 static map<string, boost::shared_ptr<deadline_timer> > deadlineTimers;
@@ -659,6 +660,7 @@ void StartRPCThreads()
     rpc_worker_group = new boost::thread_group();
     for (int i = 0; i < GetArg("-rpcthreads", 4); i++)
         rpc_worker_group->create_thread(boost::bind(&asio::io_service::run, rpc_io_service));
+    fRPCRunning = true;
 }
 
 void StartDummyRPCThread()
@@ -671,12 +673,15 @@ void StartDummyRPCThread()
         rpc_dummy_work = new asio::io_service::work(*rpc_io_service);
         rpc_worker_group = new boost::thread_group();
         rpc_worker_group->create_thread(boost::bind(&asio::io_service::run, rpc_io_service));
+        fRPCRunning = true;
     }
 }
 
 void StopRPCThreads()
 {
     if (rpc_io_service == NULL) return;
+    // Set this to false first, so that longpolling loops will exit when woken up
+    fRPCRunning = false;
 
     // First, cancel all timers and acceptors
     // This is not done automatically by ->stop(), and in some cases the destructor of
@@ -698,6 +703,7 @@ void StopRPCThreads()
     deadlineTimers.clear();
 
     rpc_io_service->stop();
+    cvBlockChange.notify_all();
     if (rpc_worker_group != NULL)
         rpc_worker_group->join_all();
     delete rpc_dummy_work; rpc_dummy_work = NULL;
@@ -706,6 +712,11 @@ void StopRPCThreads()
     delete rpc_io_service; rpc_io_service = NULL;
 }
 
+bool IsRPCRunning()
+{
+    return fRPCRunning;
+}
+
 void RPCRunHandler(const boost::system::error_code& err, boost::function<void(void)> func)
 {
     if (!err)

src/rpcserver.h

@@ -40,6 +40,8 @@ void StartRPCThreads();
 void StartDummyRPCThread();
 /* Stop RPC threads */
 void StopRPCThreads();
+/* Query whether RPC is running */
+bool IsRPCRunning();
 
 /*
   Type-check arguments; throws JSONRPCError if wrong type given. Does not check that

src/sync.h

@@ -84,6 +84,9 @@ typedef AnnotatedMixin<boost::recursive_mutex> CCriticalSection;
 /** Wrapped boost mutex: supports waiting but not recursive locking */
 typedef AnnotatedMixin<boost::mutex> CWaitableCriticalSection;
 
+/** Just a typedef for boost::condition_variable, can be wrapped later if desired */
+typedef boost::condition_variable CConditionVariable;
+
 #ifdef DEBUG_LOCKORDER
 void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false);
 void LeaveCritical();