Added RPC method getdsproofscore

Summary
---

This new method is intended to address and close Bitcoin-ABC#307.
The new RPC takes a txid as its only argument and
returns a number from `0.0` to `1.0`.

- `1.0` indicates no dsproofs exist for this tx or for any of its ancestors,
  but that it and all in-mempool ancestors **can** have a dsproof,
  so confidence should be high that the tx is ok.
- `0.0` indicates that either this tx or one of its ancestors has a dsproof,
  or that it or one of its in-mempool ancestors can never have a proof
  (not P2PKH) -- so confidence should be low.
- `0.25` is returned in the case where the tx has so many mempool ancestors
  that no conclusive determination could be made (but the ones that were
  checked are ok).
- If `txid` doesn't exist in the mempool, the usual `JSONRPCError` is thrown
  (similar to how other RPCs work).

Code Changed
---

- Refactored out the code from `compressor.cpp` that checked for p2pkh and p2sh
  and ensured this code (which was somewhat duplicated) all lives in `CScript`
  and is publicly accessible.
- Added a static function `checkIsProofPossibleForAllInputsOfTx` to the
  `DoubleSpendProof` class, which basically answers the question: "is this tx
  itself eligible for a double-spend proof"?
- Added the `getdsproofscore` RPC method to `rpc/dsproof.cpp`
- Modified `CTxMemPool::recursiveDSPSearch` to be able to answer the "score"
  question we need for the `getdsproofscore` RPC.
   - Refactored `CTxMemPool::recursiveDSPSearch` and its helper to be a bit
     faster by working with `txiter` directly.
   - Added an additional constraint: in addition to the 1000 max-deep ancestor
     chain constraint, we also cannot examine more than 20k total ancestors.
     This ensures the RPC doesn't go out to lunch for too long and has a finite
     execution time that is bounded nicely, even in pathological cases.
     In this corner case an "inconclusive" score of 0.25 is returned by the RPC.
- Fixup: `compressor.cpp` had questionable/UB use of `memcpy`
  - Fixed to not write to `uint160` directly as if it were a simple C array ..
    but rather write to its .begin() using std::memcpy. This is not UB,
    whereas the previous code technically was.
  - Also renamed `memcpy` -> `std::memcpy`, and included the right header.
- Added C++ unit tests for: `CScript::IsPayToPubKeyHash` and
  `DoubleSpendProof::checkIsProofPossibleForAllInputsOfTx`
- Added Python functional testing of `getdsproofscore`

Test Plan
---

- `ninja all check && test/functional/test_runner.py bchn-rpc-dsproof`

doc/release-notes.md

@@ -41,7 +41,14 @@ or `-doublespendproof=0`.
 
 ## New RPC methods
 
-...
+### `getdsproofscore`
+
+`getdsproofscore` returns a double-spend confidence score for a
+mempool transaction.
+
+Please refer to the documentation pages for
+[getdsproofscore](https://docs.bitcoincashnode.org/doc/json-rpc/getdsproofscore/)
+for details about additional arguments and the returned data.
 
 ## Low-level RPC changes
 

src/compressor.cpp

@@ -9,6 +9,8 @@
 #include <pubkey.h>
 #include <script/standard.h>
 
+#include <cstring>
+
 /*
  * These check for scripts for which a special case with a shorter encoding is
  * defined. They are implemented separately from the CScript test, as these test
@@ -18,19 +20,18 @@
  */
 
 static bool IsToKeyID(const CScript &script, CKeyID &hash) {
-    if (script.size() == 25 && script[0] == OP_DUP && script[1] == OP_HASH160 &&
-        script[2] == 20 && script[23] == OP_EQUALVERIFY &&
-        script[24] == OP_CHECKSIG) {
-        memcpy(&hash, &script[3], 20);
+    if (script.IsPayToPubKeyHash()) {
+        static_assert(CKeyID::size() == 20);
+        std::memcpy(&*hash.begin(), &script[3], 20);
         return true;
     }
     return false;
 }
 
 static bool IsToScriptID(const CScript &script, CScriptID &hash) {
-    if (script.size() == 23 && script[0] == OP_HASH160 && script[1] == 20 &&
-        script[22] == OP_EQUAL) {
-        memcpy(&hash, &script[2], 20);
+    if (script.IsPayToScriptHash()) {
+        static_assert(CScriptID::size() == 20);
+        std::memcpy(&*hash.begin(), &script[2], 20);
         return true;
     }
     return false;
@@ -56,20 +57,20 @@ bool CompressScript(const CScript &script, std::vector<uint8_t> &out) {
     if (IsToKeyID(script, keyID)) {
         out.resize(21);
         out[0] = 0x00;
-        memcpy(&out[1], &keyID, 20);
+        std::memcpy(&out[1], &*keyID.begin(), 20);
         return true;
     }
     CScriptID scriptID;
     if (IsToScriptID(script, scriptID)) {
         out.resize(21);
         out[0] = 0x01;
-        memcpy(&out[1], &scriptID, 20);
+        std::memcpy(&out[1], &*scriptID.begin(), 20);
         return true;
     }
     CPubKey pubkey;
     if (IsToPubKey(script, pubkey)) {
         out.resize(33);
-        memcpy(&out[1], &pubkey[1], 32);
+        std::memcpy(&out[1], &pubkey[1], 32);
         if (pubkey[0] == 0x02 || pubkey[0] == 0x03) {
             out[0] = pubkey[0];
             return true;
@@ -99,38 +100,38 @@ bool DecompressScript(CScript &script, unsigned int nSize,
             script[0] = OP_DUP;
             script[1] = OP_HASH160;
             script[2] = 20;
-            memcpy(&script[3], in.data(), 20);
+            std::memcpy(&script[3], in.data(), 20);
             script[23] = OP_EQUALVERIFY;
             script[24] = OP_CHECKSIG;
             return true;
         case 0x01:
             script.resize(23);
             script[0] = OP_HASH160;
             script[1] = 20;
-            memcpy(&script[2], in.data(), 20);
+            std::memcpy(&script[2], in.data(), 20);
             script[22] = OP_EQUAL;
             return true;
         case 0x02:
         case 0x03:
             script.resize(35);
             script[0] = 33;
             script[1] = nSize;
-            memcpy(&script[2], in.data(), 32);
+            std::memcpy(&script[2], in.data(), 32);
             script[34] = OP_CHECKSIG;
             return true;
         case 0x04:
         case 0x05:
             uint8_t vch[33] = {};
             vch[0] = nSize - 2;
-            memcpy(&vch[1], in.data(), 32);
+            std::memcpy(&vch[1], in.data(), 32);
             CPubKey pubkey(&vch[0], &vch[33]);
             if (!pubkey.Decompress()) {
                 return false;
             }
             assert(pubkey.size() == 65);
             script.resize(67);
             script[0] = 65;
-            memcpy(&script[1], pubkey.begin(), 65);
+            std::memcpy(&script[1], pubkey.data(), 65);
             script[66] = OP_CHECKSIG;
             return true;
     }

src/dsproof/dsproof.h

@@ -61,6 +61,22 @@ class DoubleSpendProof
     //! (implemented in dsproof_validate.cpp)
     Validity validate(const CTxMemPool &mempool, CTransactionRef spendingTx = {}) const EXCLUSIVE_LOCKS_REQUIRED(cs_main);
 
+    //! This *must* be called with cs_main and mempool.cs already held!
+    //!
+    //! Checks whether a tx is compatible with dsproofs and/or whether
+    //! it can have a double-spend proof generated for it for all of its
+    //! inputs.
+    //!
+    //! For now this basically just checks:
+    //!
+    //! 1. That all inputs to `tx` have "known" Coins (that is, prevouts
+    //!    refering to txs in the mempool or confirmed in the blockchain).
+    //! 2. All inputs to `tx` are spends from p2pkh addresses.
+    //!
+    //! If either of the above are false, this will return false.
+    //!
+    static bool checkIsProofPossibleForAllInputsOfTx(const CTxMemPool &mempool, const CTransaction &tx) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
+
     const TxId & prevTxId() const { return m_outPoint.GetTxId(); }
     uint32_t prevOutIndex() const { return m_outPoint.GetN(); }
     const COutPoint & outPoint() const { return m_outPoint; }

src/dsproof/dsproof_validate.cpp

@@ -159,3 +159,31 @@ auto DoubleSpendProof::validate(const CTxMemPool &mempool, CTransactionRef spend
     }
     return Valid;
 }
+
+/* static */ bool DoubleSpendProof::checkIsProofPossibleForAllInputsOfTx(const CTxMemPool &mempool, const CTransaction &tx)
+{
+    AssertLockHeld(cs_main);
+    AssertLockHeld(mempool.cs);
+
+    if (tx.vin.empty() || tx.IsCoinBase()) {
+        return false;
+    }
+
+    const CCoinsViewMemPool view(pcoinsTip.get(), mempool); // this checks both mempool coins and confirmed coins
+
+    // Check all inputs
+    for (const auto & txin : tx.vin) {
+        Coin coin;
+        if (!view.GetCoin(txin.prevout, coin)) {
+            // if the Coin this tx spends is missing then either this tx just got mined or our mempool + blockchain
+            // view just doesn't have the coin.
+            return false;
+        }
+        if (!coin.GetTxOut().scriptPubKey.IsPayToPubKeyHash()) {
+            // For now, dsproof only supports P2PKH
+            return false;
+        }
+    }
+
+    return true;
+}

src/rpc/dsproof.cpp

@@ -385,13 +385,70 @@ static UniValue getdsprooflist(const Config &,
     return ret;
 }
 
+static UniValue getdsproofscore(const Config &,
+                                const JSONRPCRequest &request) {
+    if (request.fHelp || request.params.size() != 1) {
+        throw std::runtime_error(
+            RPCHelpMan{
+                "getdsproofscore",
+                "\nReturn a double-spend confidence score for a mempool transaction.\n",
+                {{"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "",
+                  "The in-memory txid to query."},
+                },
+                RPCResults{
+                    RPCResult{"n          (numeric) A value from 0.0 to 1.0, with 1.0 indicating that the\n"
+                              "           transaction in question has no current dsproofs for it or any of its\n"
+                              "           mempool ancestors, but that a future dsproof is possible. Confidence\n"
+                              "           that this transaction has no known double-spends is relatively high.\n"
+                              "\n"
+                              "           A value of 0.0 indicates that the tx in question or one of its\n"
+                              "           mempool ancestors has a dsproof, or that it or one of its mempool\n"
+                              "           ancestors does not support dsproofs (not P2PKH), so confidence in\n"
+                              "           this tx should be low.\n"
+                              "\n"
+                              "           A value of 0.25 indicates that up to the first 20,000 ancestors were\n"
+                              "           checked and all have no proofs but *can* have proofs. Since the tx\n"
+                              "           in question has a very large mempool ancestor set, double-spend\n"
+                              "           confidence should be considered medium-to-low. (This value may also\n"
+                              "           be returned for transactions which exceed depth 1,000 in an\n"
+                              "           unconfirmed ancestor chain).\n"},
+                },
+                RPCExamples{HelpExampleCli("getdsproofscore", "d3aac244e46f4bc5e2140a07496a179624b42d12600bfeafc358154ec89a720c") +
+                            HelpExampleRpc("getdsproofscore", "d3aac244e46f4bc5e2140a07496a179624b42d12600bfeafc358154ec89a720c")}
+            }.ToStringWithResultsAndExamples());
+    }
+
+    ThrowIfDisabled(); // don't proceed if the subsystem was disabled with -doublespendproof=0
+
+    RPCTypeCheck(request.params, {UniValue::VSTR});
+
+    // lookup by txid only
+    const TxId txId{ParseHashV(request.params[0], "txid")};
+
+    double score{};
+
+    try {
+        g_mempool.recursiveDSProofSearch(txId, nullptr, &score);
+    } catch (const CTxMemPool::RecursionLimitReached &e) {
+        // ok, score will be set for us correctly anyway in this case
+        LogPrint(BCLog::DSPROOF, "getdsproofscore (txid: %s) caught exception: %s\n", txId.ToString(), e.what());
+    }
+
+    if (score < 0.0) {
+        /* A score of <0.0 means txid not found */
+        throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool");
+    }
+
+    return score;
+}
 
 // clang-format off
 static const ContextFreeRPCCommand commands[] = {
     //  category            name                      actor (function)        argNames
     //  ------------------- ------------------------  ----------------------  ----------
     { "blockchain",         "getdsproof",             getdsproof,             {"dspid|txid|outpoint", "verbosity|verbose", "recursive"} },
     { "blockchain",         "getdsprooflist",         getdsprooflist,         {"verbosity|verbose", "include_orphans"} },
+    { "blockchain",         "getdsproofscore",        getdsproofscore,        {"txid"} },
 };
 // clang-format on
 

src/script/script.cpp

@@ -373,6 +373,13 @@ bool CScript::IsPayToScriptHash() const {
             (*this)[1] == 0x14 && (*this)[22] == OP_EQUAL);
 }
 
+bool CScript::IsPayToPubKeyHash() const {
+    // Extra-fast test for P2PKH CScripts:
+    return size() == 25 && (*this)[0] == OP_DUP && (*this)[1] == OP_HASH160
+            && (*this)[2] == 20 && (*this)[23] == OP_EQUALVERIFY
+            && (*this)[24] == OP_CHECKSIG;
+}
+
 bool CScript::IsCommitment(const std::vector<uint8_t> &data) const {
     // To ensure we have an immediate push, we limit the commitment size to 64
     // bytes. In addition to the data themselves, we have 2 extra bytes:

src/script/script.h

@@ -543,6 +543,7 @@ class CScript : public CScriptBase {
     }
 
     bool IsPayToScriptHash() const;
+    bool IsPayToPubKeyHash() const;
     bool IsCommitment(const std::vector<uint8_t> &data) const;
     bool IsWitnessProgram(int &version, std::vector<uint8_t> &program) const;
     bool IsWitnessProgram() const;

src/test/dsproof_tests.cpp

@@ -275,9 +275,34 @@ BOOST_FIXTURE_TEST_CASE(dsproof_doublespend_mempool, TestChain100Setup) {
     const CScript scriptPubKey = GetScriptForDestination(coinbaseKey.GetPubKey().GetID());
     const size_t firstTxIdx = m_coinbase_txns.size();
 
+    // we were given a blockchain that mines to a p2pk address --
+    // check that txs that spend those cannot have dsproofs
+    BOOST_CHECK(!m_coinbase_txns.empty());
+    for (const auto & tx : m_coinbase_txns) {
+        LOCK2(cs_main, g_mempool.cs);
+        // belt-and-suspenders check that coinbase tx cannot have double spend proofs
+        BOOST_CHECK(!DoubleSpendProof::checkIsProofPossibleForAllInputsOfTx(g_mempool, *tx));
+        CMutableTransaction spend;
+        spend.nVersion = 1;
+        spend.vin.resize(1);
+        spend.vin[0].prevout = COutPoint(tx->GetId(), 0);
+        spend.vout.resize(1);
+        spend.vout[0].nValue = int64_t(GetRand(1'000)) * CENT;
+        spend.vout[0].scriptPubKey = scriptPubKey;
+        // Sign:
+        const auto ok = SignSignature(provider, *tx, spend, 0, SigHashType().withForkId());
+        BOOST_CHECK(ok);
+        // Also a tx spending a p2pk cannot have a dsproof
+        BOOST_CHECK(!DoubleSpendProof::checkIsProofPossibleForAllInputsOfTx(g_mempool, CTransaction{spend}));
+    }
+
+    // next, mine a bunch of blocks that send coinbase to p2pkh
     for (int i = 0; i < COINBASE_MATURITY*2 + 1; ++i) {
         const CBlock b = CreateAndProcessBlock({}, scriptPubKey);
         m_coinbase_txns.push_back(b.vtx[0]);
+        LOCK2(cs_main, g_mempool.cs);
+        // belt-and-suspenders check that coinbase tx cannot have double spend proofs
+        BOOST_CHECK(!DoubleSpendProof::checkIsProofPossibleForAllInputsOfTx(g_mempool, *m_coinbase_txns.back()));
     }
 
     // Some code-paths below need locks held
@@ -318,6 +343,8 @@ BOOST_FIXTURE_TEST_CASE(dsproof_doublespend_mempool, TestChain100Setup) {
             auto [ok, state] = ToMemPool(spend1);
             BOOST_CHECK(ok);
             BOOST_CHECK(state.IsValid());
+            // p2pkh can have dsproof
+            BOOST_CHECK(DoubleSpendProof::checkIsProofPossibleForAllInputsOfTx(g_mempool, CTransaction(spend1)));
         }
         // Add second tx to mempool, check that it is rejected and that the dsproof generated is what we expect
         {

src/test/script_p2sh_tests.cpp

@@ -224,13 +224,15 @@ BOOST_AUTO_TEST_CASE(is) {
     CScript p2sh;
     p2sh << OP_HASH160 << ToByteVector(dummy) << OP_EQUAL;
     BOOST_CHECK(p2sh.IsPayToScriptHash());
+    BOOST_CHECK(!p2sh.IsPayToPubKeyHash());
 
     // Not considered pay-to-script-hash if using one of the OP_PUSHDATA
     // opcodes:
     static const uint8_t direct[] = {OP_HASH160, 20, 0, 0, 0, 0, 0,       0,
                                      0,          0,  0, 0, 0, 0, 0,       0,
                                      0,          0,  0, 0, 0, 0, OP_EQUAL};
     BOOST_CHECK(CScript(direct, direct + sizeof(direct)).IsPayToScriptHash());
+    BOOST_CHECK(!CScript(direct, direct + sizeof(direct)).IsPayToPubKeyHash());
     static const uint8_t pushdata1[] = {OP_HASH160, OP_PUSHDATA1,
                                         20,         0,
                                         0,          0,
@@ -245,6 +247,8 @@ BOOST_AUTO_TEST_CASE(is) {
                                         0,          OP_EQUAL};
     BOOST_CHECK(
         !CScript(pushdata1, pushdata1 + sizeof(pushdata1)).IsPayToScriptHash());
+    BOOST_CHECK(
+        !CScript(pushdata1, pushdata1 + sizeof(pushdata1)).IsPayToPubKeyHash());
     static const uint8_t pushdata2[] = {OP_HASH160, OP_PUSHDATA2,
                                         20,         0,
                                         0,          0,
@@ -260,6 +264,8 @@ BOOST_AUTO_TEST_CASE(is) {
                                         OP_EQUAL};
     BOOST_CHECK(
         !CScript(pushdata2, pushdata2 + sizeof(pushdata2)).IsPayToScriptHash());
+    BOOST_CHECK(
+        !CScript(pushdata2, pushdata2 + sizeof(pushdata2)).IsPayToPubKeyHash());
     static const uint8_t pushdata4[] = {OP_HASH160, OP_PUSHDATA4,
                                         20,         0,
                                         0,          0,
@@ -276,22 +282,38 @@ BOOST_AUTO_TEST_CASE(is) {
                                         OP_EQUAL};
     BOOST_CHECK(
         !CScript(pushdata4, pushdata4 + sizeof(pushdata4)).IsPayToScriptHash());
+    BOOST_CHECK(
+        !CScript(pushdata4, pushdata4 + sizeof(pushdata4)).IsPayToPubKeyHash());
 
     CScript not_p2sh;
     BOOST_CHECK(!not_p2sh.IsPayToScriptHash());
+    BOOST_CHECK(!not_p2sh.IsPayToPubKeyHash());
 
     not_p2sh.clear();
     not_p2sh << OP_HASH160 << ToByteVector(dummy) << ToByteVector(dummy)
              << OP_EQUAL;
     BOOST_CHECK(!not_p2sh.IsPayToScriptHash());
+    BOOST_CHECK(!not_p2sh.IsPayToPubKeyHash());
 
     not_p2sh.clear();
     not_p2sh << OP_NOP << ToByteVector(dummy) << OP_EQUAL;
     BOOST_CHECK(!not_p2sh.IsPayToScriptHash());
+    BOOST_CHECK(!not_p2sh.IsPayToPubKeyHash());
 
     not_p2sh.clear();
     not_p2sh << OP_HASH160 << ToByteVector(dummy) << OP_CHECKSIG;
     BOOST_CHECK(!not_p2sh.IsPayToScriptHash());
+    BOOST_CHECK(!not_p2sh.IsPayToPubKeyHash());
+
+    // lastly, check p2pkh
+    CScript p2pkh;
+    p2pkh << OP_DUP << OP_HASH160 << ToByteVector(dummy) << OP_EQUALVERIFY << OP_CHECKSIG;
+    BOOST_CHECK(!p2pkh.IsPayToScriptHash());
+    BOOST_CHECK(p2pkh.IsPayToPubKeyHash());
+    // break p2pkh by erasing the 10th byte
+    p2pkh.erase(p2pkh.begin() + 10);
+    BOOST_CHECK(!p2pkh.IsPayToScriptHash());
+    BOOST_CHECK(!p2pkh.IsPayToPubKeyHash());
 }
 
 BOOST_AUTO_TEST_CASE(switchover) {