Merge #260: Revamp peg-in transactions

3ff4aab update unit test (Gregory Sanders)
6cc380a don't require claim_script to be witness program for raw claim pegin (Gregory Sanders)
da6960d validation variables changed to not assume witness program for pegin (Gregory Sanders)
2940c4e remove tabs from IsValidPegin (Gregory Sanders)
b401ff0 change explanation of policy asset seeding in python tutorial (Gregory Sanders)
7dc4f9d remove CNumScript from pegin witness logic (Gregory Sanders)
abafbac change rpc test witness_program to claim_script (Gregory Sanders)
851a30b Don't restrict pegin witness to 'witness program' definition (Gregory Sanders)
13c3dad de-dupe calculate_contract code (Gregory Sanders)
6534bd3 Add is_pegin response for raw transaction RPCs (Gregory Sanders)
d4d00da basic functional test for peg-in reorgs in sidechain (Gregory Sanders)
4aad513 Add unit tests for peg-in authorization (Gregory Sanders)
be735cf re-add pegging python test (Gregory Sanders)
3ba146d update assets tutorial to support new peg-in format (Gregory Sanders)
6e8e6b2 add wallet support for pegging functionality (Gregory Sanders)
7633839 add raw rpc support for peg-in functionality (Gregory Sanders)
ff5e9cd mempool and policy support for peg-in inputs (Gregory Sanders)
6bad4e9 add blocks that have pegins rejected to the reconsider queue (Gregory Sanders)
0b9c5eb begin consensus support for peg-ins (Gregory Sanders)
34d7cd0 add pegin witness validation and extraction helper functions (Gregory Sanders)
16973cd add pegin_witness to witness inputs (Gregory Sanders)
36b3ab1 add serialization flag support for peg-in inputs (Gregory Sanders)
9a9ac25 genesis block has actual fee issuance in regtest (Gregory Sanders)
65ed777 remove legacy peg-in/withdrawlocks (Gregory Sanders)

contrib/assets_tutorial/assets_tutorial.py

@@ -46,7 +46,6 @@ def sync_all(e1, e2):
         time.sleep(1)
     return
 
-
 ## Preparations
 
 # Make data directories for each daemon
@@ -91,20 +90,19 @@ def sync_all(e1, e2):
 # Alternatively, you can set validatepegin=0 in their configs and not
 # run the bitcoin node, but it is necessary for fully validating the two way peg.
 
-# At beginning "immature balance" holds all funds until genesis is mature
-# Regtest chain starts with 21M elements coins as OP_TRUE which the wallet
-# understands. This is useful for testing basic functionality.
-# In Elements there is no block subsidy. All 21M output value in a mainnet
-# pegged sidechain are locked for pegging.
+# Regtest chain starts with 21M bitcoins as OP_TRUE which the wallet
+# understands. This is useful for testing basic functionality and for
+# blockchains that have no pegging functionality. A fee currency is required
+# for anti-DoS purposes as well as asset issuance, which consumes inputs for entropy.
+# In Elements there is no block subsidy. In a production sidechain it can
+# be configured to start with no outputs, necessitating peg-in functionality
+# for asset issuance.
 e1.getwalletinfo()
 
 # In regtest mining "target" is OP_TRUE since we have not set `-signblockscript` argument
 # Generate simply works.
 e1.generate(101)
 sync_all(e1, e2)
-# Now we have 21M OP_TRUE value in each Elements wallet.
-e1.getwalletinfo()
-e2.getwalletinfo()
 
 ######## WALLET ###########
 
@@ -419,12 +417,6 @@ def sync_all(e1, e2):
 bitcoin.generate(101)
 sync_all(e1, e2)
 
-# We have to lock up some of the funds first. Regtest(what we're running) has all funds as OP_TRUE
-# but this is not the case in testnet/production. Doesn't matter where we send it
-# inside Bitcoin, this is just a hack to lock some funds up.
-# We can immediately grab this output from the mempool on subsequent steps
-e1.sendtomainchain(bitcoin.getnewaddress(), 50)
-
 # Now we can actually start pegging in. Examine the pegin address fields
 e1.getpeginaddress()
 # Changes each time as it's a new sidechain address as well as new "tweak" for the watchmen keys
@@ -448,7 +440,7 @@ def sync_all(e1, e2):
 raw = bitcoin.getrawtransaction(txid)
 
 # Attempt claim!
-claimtxid = e1.claimpegin(raw, proof, addrs["sidechain_address"])
+claimtxid = e1.claimpegin(raw, proof, addrs["claim_script"])
 sync_all(e1, e2)
 
 # Other node should accept to mempool and mine

contrib/assets_tutorial/assets_tutorial.sh

@@ -39,18 +39,11 @@ e2-dae
 # run the bitcoin node, but it is necessary for the two way peg parts of
 # this tutorial.
 
-# Prime the chain, see "immature balance" holds all funds until genesis is mature
-e1-cli getwalletinfo
-
-# Mining for now is OP_TRUE
-e1-cli generate 101
-
-# Now we have 21M OP_TRUE value
+# We have 21M OP_TRUE value in each wallet
+# This is useful for testing and non-sidechain applications of Elements
 e1-cli getwalletinfo
 e2-cli getwalletinfo
 
-# Primed and ready
-
 ######## WALLET ###########
 
 
@@ -311,19 +304,11 @@ e2-dae $FEDPEGARG
 e1-cli generate 101
 b-cli generate 101
 
-# We have to lock up some of the funds first. Regtest(what we're running) has all funds as OP_TRUE
-# but this is not the case in testnet/production. Doesn't matter where we send it
-# inside Bitcoin, this is just a hack to lock some funds up.
-e1-cli sendtomainchain $(b-cli getnewaddress) 50
-
-# Mature the pegout
-e1-cli generate 101
-
 # Now we can actually start pegging in. Examine the pegin address fields
 e1-cli getpeginaddress
 # Changes each time as it's a new sidechain address as well as new "tweak" for the watchmen keys
 # mainchain_address : where you send your bitcoin from Bitcoin network
-# sidechain_address : where the bitcoin will end up on the sidechain after pegging in
+# claim_script: what script will have to be satisfied to spent the peg-in input
 
 # Each call of this takes the pubkeys defined in the config file, adds a random number to them
 # that is essetially the hash of the sidechain_address and other information,
@@ -335,7 +320,7 @@ e1-cli getpeginaddress
 ADDRS=$(e1-cli getpeginaddress)
 
 MAINCHAIN=$(echo $ADDRS | python3 -c "import sys, json; print(json.load(sys.stdin)['mainchain_address'])")
-SIDECHAIN=$(echo $ADDRS | python3 -c "import sys, json; print(json.load(sys.stdin)['sidechain_address'])")
+SIDECHAIN=$(echo $ADDRS | python3 -c "import sys, json; print(json.load(sys.stdin)['claim_script'])")
 
 #Send funds to unique watchmen P2SH address
 TXID=$(b-cli sendtoaddress $MAINCHAIN 1)

qa/rpc-tests/confidential_transactions.py

@@ -222,6 +222,10 @@ def run_test(self):
 
         print("Assets tests...")
 
+        # Bitcoin is the first issuance
+        assert_equal(self.nodes[0].listissuances()[0]["assetlabel"], "bitcoin")
+        assert_equal(len(self.nodes[0].listissuances()), 1)
+
         # Unblinded issuance of asset
         issued = self.nodes[0].issueasset(1, 1, False)
         assert_equal(self.nodes[0].getwalletinfo()["balance"][issued["asset"]], 1)
@@ -351,13 +355,13 @@ def run_test(self):
         addr2 = txdet2[len(txdet2)-1]["address"]
         addr3 = txdet3[len(txdet3)-1]["address"]
 
-        assert_equal(len(self.nodes[0].listissuances()), 5);
+        assert_equal(len(self.nodes[0].listissuances()), 6);
         self.nodes[0].importaddress(addr1)
         self.nodes[0].importaddress(addr2)
         self.nodes[0].importaddress(addr3)
 
         issuances = self.nodes[0].listissuances()
-        assert_equal(len(issuances), 8)
+        assert_equal(len(issuances), 9)
 
         for issue in issuances:
             if issue['txid'] == redata1["txid"] and issue['vin'] == redata1["vin"]:

qa/rpc-tests/pegging.py

@@ -14,6 +14,22 @@
 print(sys.argv[1])
 print(sys.argv[2])
 
+# Sync mempool, make a block, sync blocks
+def sync_all(sidechain, sidechain2):
+    timeout = 20
+    while len(sidechain.getrawmempool()) != len(sidechain2.getrawmempool()):
+        time.sleep(1)
+        timeout -= 1
+        if timeout == 0:
+            raise Exception("Peg-in has failed to propagate.")
+    block = sidechain2.generate(1)
+    while sidechain.getblockcount() != sidechain2.getblockcount():
+        time.sleep(1)
+        timeout -= 1
+        if timeout == 0:
+            raise Exception("Blocks are not propagating.")
+    return block
+
 fedpeg_key="cPxqWyf1HDGpGFH1dnfjz8HbiWxvwG8WXyetbuAiw4thKXUdXLpR"
 fedpeg_pubkey="512103dff4923d778550cc13ce0d887d737553b4b58f4e8e886507fc39f5e447b2186451ae"
 
@@ -88,7 +104,7 @@
 
 try:
 
-    # Default is 8, meaning 8+2 confirms for mempool acceptance normally
+    # Default is 8, meaning 8+2 confirms for wallet acceptance normally
     # this will require 10+2.
     sidechain_args = " -peginconfirmationdepth=10 "
 
@@ -116,16 +132,8 @@
 
     addr = bitcoin.getnewaddress()
 
-    # Lockup some funds to unlock later
-    sidechain.sendtomainchain(addr, 50)
-    # Tests withdrawlock tracking in database
-    sidechain.generate(1)
-    # Tests withdrawlock in mempool
-    sidechain.sendtomainchain(addr, 50)
-
     addrs = sidechain.getpeginaddress()
     txid1 = bitcoin.sendtoaddress(addrs["mainchain_address"], 24)
-    txid2 = bitcoin.sendtoaddress(addrs["mainchain_address"], 24)
     # 10+2 confirms required to get into mempool and confirm
     bitcoin.generate(11)
     time.sleep(2)
@@ -137,59 +145,55 @@
         pegtxid = sidechain.claimpegin(raw, proof)
         raise Exception("Peg-in should not mature enough yet, need another block.")
     except JSONRPCException as e:
-        assert("Withdraw proof validation failed" in e.error["message"])
+        assert("Peg-in Bitcoin transaction needs more confirmations to be sent." in e.error["message"])
         pass
 
     # Should fail due to non-matching wallet address
     try:
         pegtxid = sidechain.claimpegin(raw, proof, sidechain.getnewaddress())
-        raise Exception("Peg-in with non-matching address should fail.")
+        raise Exception("Peg-in with non-matching claim_script should fail.")
     except JSONRPCException as e:
-        assert("Failed to find output in bitcoinTx to the mainchain_address" in e.error["message"])
+        assert("Given claim_script does not match the given Bitcoin transaction." in e.error["message"])
         pass
 
     # 12 confirms allows in mempool
     bitcoin.generate(1)
-
-    timeout = 20
-    # Both should succeed via wallet lookup for address match, and when given
+    # Should succeed via wallet lookup for address match, and when given
     pegtxid1 = sidechain.claimpegin(raw, proof)
 
-    proof = bitcoin.gettxoutproof([txid2])
-    raw = bitcoin.getrawtransaction(txid2)
-    pegtxid2 = sidechain.claimpegin(raw, proof, addrs["sidechain_address"])
-
-    while len(sidechain.getrawmempool()) != len(sidechain2.getrawmempool()):
-        time.sleep(1)
-        timeout -= 1
-        if timeout == 0:
-            raise Exception("Peg-in has failed to propagate.")
-    sidechain2.generate(1)
-    while sidechain.getblockcount() != sidechain2.getblockcount():
-        time.sleep(1)
-        timeout -= 1
-        if timeout == 0:
-            raise Exception("Blocks are not propagating.")
-
+    # Will invalidate the block that confirms this transaction later
+    blockhash = sync_all(sidechain, sidechain2)
+    sidechain.generate(5)
 
     tx1 = sidechain.gettransaction(pegtxid1)
-    tx2 = sidechain.gettransaction(pegtxid2)
 
-    if "confirmations" in tx1 and tx1["confirmations"] > 0 and "confirmations" in tx2 and tx2["confirmations"] > 0:
+    if "confirmations" in tx1 and tx1["confirmations"] == 6:
         print("Peg-in is confirmed: Success!")
     else:
         raise Exception("Peg-in confirmation has failed.")
 
-    # Make a few large locks, then do many claims in mempool
-    n_locks = 10
-    n_claims = 30
+    # Look at pegin fields
+    decoded = sidechain.decoderawtransaction(tx1["hex"])
+    assert decoded["vin"][0]["is_pegin"] == True
+    assert len(decoded["vin"][0]["pegin_witness"]) > 0
+
+    # Quick reorg checks of pegs
+    sidechain.invalidateblock(blockhash[0])
+    if sidechain.gettransaction(pegtxid1)["confirmations"] != 0:
+        raise Exception("Peg-in didn't unconfirm after invalidateblock call.")
+    # Re-enters block
+    sidechain.generate(1)
+    if sidechain.gettransaction(pegtxid1)["confirmations"] != 1:
+        raise Exception("Peg-in should have one confirm on side block.")
+    sidechain.reconsiderblock(blockhash[0])
+    if sidechain.gettransaction(pegtxid1)["confirmations"] != 6:
+        raise Exception("Peg-in should be back to 6 confirms.")
+
+    # Do many claims in mempool
+    n_claims = 100
 
     print("Flooding mempool with many small claims")
     pegtxs = []
-    for i in range(n_locks):
-        # Lockup some funds to unlock later
-        sidechain.sendtomainchain(addr, 50)
-        sidechain.generate(1)
     sidechain.generate(101)
 
     for i in range(n_claims):
@@ -200,21 +204,16 @@
         raw = bitcoin.getrawtransaction(txid)
         pegtxs += [sidechain.claimpegin(raw, proof)]
 
-    sidechain.generate(1)
+    sync_all(sidechain, sidechain2)
+
+    sidechain2.generate(1)
     for pegtxid in pegtxs:
         tx = sidechain.gettransaction(pegtxid)
         if "confirmations" not in tx or tx["confirmations"] == 0:
             raise Exception("Peg-in confirmation has failed.")
 
     print("Success!")
 
-    # Testing sidechain info RPC
-    sideinfo = sidechain.getsidechaininfo()
-    assert sideinfo["fedpegscript"] == fedpeg_pubkey
-    assert sideinfo["pegged_asset"] == sidechain.dumpassetlabels()["bitcoin"]
-    assert sideinfo["min_peg_diff"] == "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
-    assert sideinfo["parent_blockhash"] == "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"
-
 except JSONRPCException as e:
         print("Pegging testing failed, aborting:")
         print(e.error)

qa/rpc-tests/wallet.py

@@ -30,19 +30,15 @@ def setup_network(self, split=False):
 
     def run_test (self):
 
-        # Check that there's no UTXO on none of the nodes
-        assert_equal(len(self.nodes[0].listunspent()), 0)
-        assert_equal(len(self.nodes[1].listunspent()), 0)
-        assert_equal(len(self.nodes[2].listunspent()), 0)
+        # Check that there's 100 UTXOs on each of the nodes
+        assert_equal(len(self.nodes[0].listunspent()), 100)
+        assert_equal(len(self.nodes[1].listunspent()), 100)
+        assert_equal(len(self.nodes[2].listunspent()), 100)
 
-        print("Mining blocks...")
-
-        self.nodes[0].generate(1)
         walletinfo = self.nodes[0].getwalletinfo()
-        assert_equal(walletinfo['immature_balance']["bitcoin"], 21000000)
-        assert("bitcoin" not in walletinfo['balance'])
+        assert_equal(walletinfo['balance']["bitcoin"], 21000000)
 
-        self.sync_all()
+        print("Mining blocks...")
         self.nodes[1].generate(101)
         self.sync_all()
 

src/Makefile.am

@@ -239,6 +239,8 @@ libbitcoin_wallet_a_SOURCES = \
   wallet/wallet.cpp \
   wallet/walletdb.cpp \
   policy/rbf.cpp \
+  primitives/bitcoin/merkleblock.cpp \
+  primitives/bitcoin/block.cpp \
   $(BITCOIN_CORE_H)
 
 # crypto primitives library
@@ -458,7 +460,7 @@ endif
 
 libelementsconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined -version-info 1:0:0 $(RELDFLAGS)
 libelementsconsensus_la_LIBADD = $(LIBSECP256K1)
-libelementsconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL -DBITCOIN_SCRIPT_NO_CALLRPC
+libelementsconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL
 libelementsconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 
 endif

src/Makefile.test.include

@@ -100,7 +100,6 @@ BITCOIN_TESTS =\
   test/hash_tests.cpp \
   test/key_tests.cpp \
   test/limitedmap_tests.cpp \
-  test/lockedutxo_tests.cpp \
   test/withdrawspent_tests.cpp \
   test/dbwrapper_tests.cpp \
   test/main_tests.cpp \
@@ -110,6 +109,7 @@ BITCOIN_TESTS =\
   test/multisig_tests.cpp \
   test/net_tests.cpp \
   test/netbase_tests.cpp \
+  test/pegin_witness_tests.cpp \
   test/pmt_tests.cpp \
   test/policyestimator_tests.cpp \
   test/pow_tests.cpp \

src/bench/mempool_eviction.cpp

@@ -16,11 +16,11 @@ static void AddTx(const CTransaction& tx, const CAmount& nFee, CTxMemPool& pool)
     unsigned int nHeight = 1;
     bool spendsCoinbase = false;
     unsigned int sigOpCost = 4;
-    std::set<std::pair<uint256, COutPoint> > setWithdrawsSpent;
+    std::set<std::pair<uint256, COutPoint> > setPeginsSpent;
     LockPoints lp;
     pool.addUnchecked(tx.GetHash(), CTxMemPoolEntry(
                                         MakeTransactionRef(tx), nFee, nTime, dPriority, nHeight,
-                                        0, spendsCoinbase, sigOpCost, lp, setWithdrawsSpent));
+                                        0, spendsCoinbase, sigOpCost, lp, setPeginsSpent));
 }
 
 // Right now this is only testing eviction performance in an extremely small

src/bench/verify_script.cpp

@@ -81,7 +81,7 @@ static void VerifyScriptBench(benchmark::State& state)
             txCredit.vout[0].scriptPubKey,
             &txSpend.wit.vtxinwit[0].scriptWitness,
             flags,
-            MutableTransactionNoWithdrawsSignatureChecker(&txSpend, 0, txCredit.vout[0].nValue),
+            MutableTransactionSignatureChecker(&txSpend, 0, txCredit.vout[0].nValue),
             &err);
         assert(err == SCRIPT_ERR_OK);
         assert(success);