From 7c929cf5bc75aeb7c39af83324821bcd97ccfba0 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 9 Aug 2016 13:34:58 +1200 Subject: [PATCH 01/22] Add support for spending keys to the basic key store --- src/Makefile.gtest.include | 1 + src/gtest/test_keystore.cpp | 26 ++++++++++++++++++++ src/keystore.cpp | 11 +++++++++ src/keystore.h | 49 +++++++++++++++++++++++++++++++++++++ src/uint252.h | 2 ++ src/zcash/Address.cpp | 6 ++--- src/zcash/Address.hpp | 6 +++-- 7 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 src/gtest/test_keystore.cpp diff --git a/src/Makefile.gtest.include b/src/Makefile.gtest.include index 5b7fa31e29d..99fdf1e754d 100644 --- a/src/Makefile.gtest.include +++ b/src/Makefile.gtest.include @@ -8,6 +8,7 @@ zcash_gtest_SOURCES = \ gtest/test_checktransaction.cpp \ gtest/test_equihash.cpp \ gtest/test_joinsplit.cpp \ + gtest/test_keystore.cpp \ gtest/test_noteencryption.cpp \ gtest/test_merkletree.cpp \ gtest/test_circuit.cpp \ diff --git a/src/gtest/test_keystore.cpp b/src/gtest/test_keystore.cpp new file mode 100644 index 00000000000..987bdd13acc --- /dev/null +++ b/src/gtest/test_keystore.cpp @@ -0,0 +1,26 @@ +#include + +#include "keystore.h" +#include "zcash/Address.hpp" + +TEST(keystore_tests, store_and_retrieve_spending_key) { + CBasicKeyStore keyStore; + + std::set addrs; + keyStore.GetPaymentAddresses(addrs); + ASSERT_EQ(0, addrs.size()); + + auto sk = libzcash::SpendingKey::random(); + keyStore.AddSpendingKey(sk); + + auto addr = sk.address(); + ASSERT_TRUE(keyStore.HaveSpendingKey(addr)); + + libzcash::SpendingKey keyOut; + keyStore.GetSpendingKey(addr, keyOut); + ASSERT_EQ(sk, keyOut); + + keyStore.GetPaymentAddresses(addrs); + ASSERT_EQ(1, addrs.size()); + ASSERT_EQ(1, addrs.count(addr)); +} diff --git a/src/keystore.cpp b/src/keystore.cpp index 3bae24b7b94..376654e7f5b 100644 --- a/src/keystore.cpp +++ b/src/keystore.cpp @@ -23,6 +23,10 @@ bool CKeyStore::AddKey(const CKey &key) { return AddKeyPubKey(key, key.GetPubKey()); } +bool CKeyStore::AddSpendingKey(const libzcash::SpendingKey &key) { + return AddSpendingKeyPaymentAddress(key, key.address()); +} + bool CBasicKeyStore::AddKeyPubKey(const CKey& key, const CPubKey &pubkey) { LOCK(cs_KeyStore); @@ -83,3 +87,10 @@ bool CBasicKeyStore::HaveWatchOnly() const LOCK(cs_KeyStore); return (!setWatchOnly.empty()); } + +bool CBasicKeyStore::AddSpendingKeyPaymentAddress(const libzcash::SpendingKey& key, const libzcash::PaymentAddress &address) +{ + LOCK(cs_KeyStore); + mapSpendingKeys[address] = key; + return true; +} diff --git a/src/keystore.h b/src/keystore.h index 4a4b6d20afe..bbd04f235c8 100644 --- a/src/keystore.h +++ b/src/keystore.h @@ -11,6 +11,7 @@ #include "script/script.h" #include "script/standard.h" #include "sync.h" +#include "zcash/Address.hpp" #include #include @@ -44,11 +45,21 @@ class CKeyStore virtual bool RemoveWatchOnly(const CScript &dest) =0; virtual bool HaveWatchOnly(const CScript &dest) const =0; virtual bool HaveWatchOnly() const =0; + + //! Add a spending key to the store. + virtual bool AddSpendingKeyPaymentAddress(const libzcash::SpendingKey &key, const libzcash::PaymentAddress &address) =0; + virtual bool AddSpendingKey(const libzcash::SpendingKey &key); + + //! Check whether a spending key corresponding to a given payment address is present in the store. + virtual bool HaveSpendingKey(const libzcash::PaymentAddress &address) const =0; + virtual bool GetSpendingKey(const libzcash::PaymentAddress &address, libzcash::SpendingKey& keyOut) const =0; + virtual void GetPaymentAddresses(std::set &setAddress) const =0; }; typedef std::map KeyMap; typedef std::map ScriptMap; typedef std::set WatchOnlySet; +typedef std::map SpendingKeyMap; /** Basic key store, that keeps keys in an address->secret map */ class CBasicKeyStore : public CKeyStore @@ -57,6 +68,7 @@ class CBasicKeyStore : public CKeyStore KeyMap mapKeys; ScriptMap mapScripts; WatchOnlySet setWatchOnly; + SpendingKeyMap mapSpendingKeys; public: bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey); @@ -103,6 +115,43 @@ class CBasicKeyStore : public CKeyStore virtual bool RemoveWatchOnly(const CScript &dest); virtual bool HaveWatchOnly(const CScript &dest) const; virtual bool HaveWatchOnly() const; + + bool AddSpendingKeyPaymentAddress(const libzcash::SpendingKey &key, const libzcash::PaymentAddress &address); + bool HaveSpendingKey(const libzcash::PaymentAddress &address) const + { + bool result; + { + LOCK(cs_KeyStore); + result = (mapSpendingKeys.count(address) > 0); + } + return result; + } + bool GetSpendingKey(const libzcash::PaymentAddress &address, libzcash::SpendingKey &keyOut) const + { + { + LOCK(cs_KeyStore); + SpendingKeyMap::const_iterator mi = mapSpendingKeys.find(address); + if (mi != mapSpendingKeys.end()) + { + keyOut = mi->second; + return true; + } + } + return false; + } + void GetPaymentAddresses(std::set &setAddress) const + { + setAddress.clear(); + { + LOCK(cs_KeyStore); + SpendingKeyMap::const_iterator mi = mapSpendingKeys.begin(); + while (mi != mapSpendingKeys.end()) + { + setAddress.insert((*mi).first); + mi++; + } + } + } }; typedef std::vector > CKeyingMaterial; diff --git a/src/uint252.h b/src/uint252.h index c5ab1a38067..6281e853334 100644 --- a/src/uint252.h +++ b/src/uint252.h @@ -43,6 +43,8 @@ class uint252 { uint256 inner() const { return contents; } + + friend inline bool operator==(const uint252& a, const uint252& b) { return a.contents == b.contents; } }; #endif diff --git a/src/zcash/Address.cpp b/src/zcash/Address.cpp index 446a63db157..9bb32fb6c0f 100644 --- a/src/zcash/Address.cpp +++ b/src/zcash/Address.cpp @@ -8,7 +8,7 @@ uint256 ViewingKey::pk_enc() { return ZCNoteEncryption::generate_pubkey(*this); } -ViewingKey SpendingKey::viewing_key() { +ViewingKey SpendingKey::viewing_key() const { return ViewingKey(ZCNoteEncryption::generate_privkey(*this)); } @@ -16,8 +16,8 @@ SpendingKey SpendingKey::random() { return SpendingKey(random_uint252()); } -PaymentAddress SpendingKey::address() { +PaymentAddress SpendingKey::address() const { return PaymentAddress(PRF_addr_a_pk(*this), viewing_key().pk_enc()); } -} \ No newline at end of file +} diff --git a/src/zcash/Address.hpp b/src/zcash/Address.hpp index 86e351cf7f6..36b9402a38a 100644 --- a/src/zcash/Address.hpp +++ b/src/zcash/Address.hpp @@ -22,6 +22,8 @@ class PaymentAddress { READWRITE(a_pk); READWRITE(pk_enc); } + + friend inline bool operator<(const PaymentAddress& a, const PaymentAddress& b) { return a.a_pk < b.a_pk; } }; class ViewingKey : public uint256 { @@ -38,8 +40,8 @@ class SpendingKey : public uint252 { static SpendingKey random(); - ViewingKey viewing_key(); - PaymentAddress address(); + ViewingKey viewing_key() const; + PaymentAddress address() const; }; } From f88f51f845f849cb2891418031b1b669f5a4532c Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 9 Aug 2016 23:15:22 -0700 Subject: [PATCH 02/22] Implemented RPC calls z_importkey, z_exportkey, z_getnewaddress. Modified RPC calls dumpwallet and importwallet to include spending keys. --- src/rpcclient.cpp | 3 +- src/rpcserver.cpp | 5 +- src/rpcserver.h | 4 ++ src/wallet/rpcdump.cpp | 143 +++++++++++++++++++++++++++++++++++++++ src/wallet/rpcwallet.cpp | 26 +++++++ src/wallet/wallet.cpp | 54 +++++++++++++++ src/wallet/wallet.h | 16 +++++ src/wallet/walletdb.cpp | 48 ++++++++++++- src/wallet/walletdb.h | 4 ++ 9 files changed, 300 insertions(+), 3 deletions(-) diff --git a/src/rpcclient.cpp b/src/rpcclient.cpp index 7efb7d74c81..78ae7d93057 100644 --- a/src/rpcclient.cpp +++ b/src/rpcclient.cpp @@ -97,7 +97,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "zcrawjoinsplit", 4 }, { "zcbenchmark", 1 }, { "zcbenchmark", 2 }, - { "getblocksubsidy", 0} + { "getblocksubsidy", 0}, + { "z_importkey", 1 } }; class CRPCConvertTable diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 90a255401ef..afeb88d4721 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -381,7 +381,10 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "zcrawkeygen", &zc_raw_keygen, true }, { "wallet", "zcrawjoinsplit", &zc_raw_joinsplit, true }, { "wallet", "zcrawreceive", &zc_raw_receive, true }, - { "wallet", "zcsamplejoinsplit", &zc_sample_joinsplit, true } + { "wallet", "zcsamplejoinsplit", &zc_sample_joinsplit, true }, + { "wallet", "z_getnewaddress", &z_getnewaddress, true }, + { "wallet", "z_exportkey", &z_exportkey, true }, + { "wallet", "z_importkey", &z_importkey, true } #endif // ENABLE_WALLET }; diff --git a/src/rpcserver.h b/src/rpcserver.h index d9056489102..f34914a45d0 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -243,6 +243,10 @@ extern json_spirit::Value reconsiderblock(const json_spirit::Array& params, bool extern json_spirit::Value getblocksubsidy(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value z_exportkey(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp +extern json_spirit::Value z_importkey(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp +extern json_spirit::Value z_getnewaddress(const json_spirit::Array& params, bool fHelp); // in rpcwallet.cpp + // in rest.cpp extern bool HTTPReq_REST(AcceptedConnection *conn, const std::string& strURI, diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index ab951d1d7d1..db5f36ed0e1 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -265,6 +265,32 @@ Value importwallet(const Array& params, bool fHelp) boost::split(vstr, line, boost::is_any_of(" ")); if (vstr.size() < 2) continue; + + // Let's see if the address is a valid Zcash spending key + try { + CZCSpendingKey spendingkey(vstr[0]); + libzcash::SpendingKey key = spendingkey.Get(); + libzcash::PaymentAddress addr = key.address(); + if (pwalletMain->HaveSpendingKey(addr)) { + LogPrintf("Skipping import of zaddr %s (key already present)\n", CZCPaymentAddress(addr).ToString()); + continue; + } + int64_t nTime = DecodeDumpTime(vstr[1]); + LogPrintf("Importing zaddr %s...\n", CZCPaymentAddress(addr).ToString()); + if (!pwalletMain->AddZKey(key)) { + // Something went wrong + fGood = false; + continue; + } + // Successfully imported zaddr. Now import the metadata. + pwalletMain->mapZKeyMetadata[addr].nCreateTime = nTime; + continue; + } + catch (...) { + // Not a valid spending key, so carry on and see if it's a Bitcoin style address. + } + + CBitcoinSecret vchSecret; if (!vchSecret.SetString(vstr[0])) continue; @@ -419,7 +445,124 @@ Value dumpwallet(const Array& params, bool fHelp) } } file << "\n"; + + // dump the zkeys + std::set addresses; + pwalletMain->GetPaymentAddresses(addresses); + file << "\n"; + file << "# Zkeys\n"; + file << "\n"; + for (auto addr : addresses ) { + libzcash::SpendingKey key; + if (pwalletMain->GetSpendingKey(addr, key)) { + std::string strTime = EncodeDumpTime(pwalletMain->mapZKeyMetadata[addr].nCreateTime); + file << strprintf("%s %s # zaddr=%s\n", CZCSpendingKey(key).ToString(), strTime, CZCPaymentAddress(addr).ToString()); + } + } + file << "\n"; + file << "# End of dump\n"; file.close(); return Value::null; } + + +Value z_importkey(const Array& params, bool fHelp) +{ + if (!EnsureWalletIsAvailable(fHelp)) + return Value::null; + + if (fHelp || params.size() < 1 || params.size() > 3) + throw runtime_error( + "z_importkey \"zkey\" ( \"label\" rescan )\n" + "\nAdds a zkey (as returned by z_exportkey) to your wallet.\n" + "\nArguments:\n" + "1. \"zkey\" (string, required) The zkey (see z_exportkey)\n" + "2. rescan (boolean, optional, default=true) Rescan the wallet for transactions\n" + "\nNote: This call can take minutes to complete if rescan is true.\n" + "\nExamples:\n" + "\nExport a zkey\n" + + HelpExampleCli("z_exportkey", "\"myaddress\"") + + "\nImport the zkey with rescan\n" + + HelpExampleCli("z_importkey", "\"mykey\"") + + "\nImport using a label and without rescan\n" + + HelpExampleCli("z_importkey", "\"mykey\" \"testing\" false") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("z_importkey", "\"mykey\", \"testing\", false") + ); + + LOCK2(cs_main, pwalletMain->cs_wallet); + + EnsureWalletIsUnlocked(); + + // Whether to perform rescan after import + bool fRescan = true; + if (params.size() > 1) + fRescan = params[1].get_bool(); + + string strSecret = params[0].get_str(); + CZCSpendingKey spendingkey(strSecret); + auto key = spendingkey.Get(); + auto addr = key.address(); + + { + pwalletMain->MarkDirty(); + + // Don't throw error in case a key is already there + if (pwalletMain->HaveSpendingKey(addr)) + return Value::null; + + if (!pwalletMain-> AddZKey(key)) + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding spending key to wallet"); + + pwalletMain->mapZKeyMetadata[addr].nCreateTime = 1; + + // We want to scan for transactions and notes + if (fRescan) { + pwalletMain->ScanForWalletTransactions(chainActive.Genesis(), true); + } + } + + return Value::null; +} + + +Value z_exportkey(const Array& params, bool fHelp) +{ + if (!EnsureWalletIsAvailable(fHelp)) + return Value::null; + + if (fHelp || params.size() != 1) + throw runtime_error( + "z_exportkey \"zaddr\"\n" + "\nReveals the zkey corresponding to 'zaddr'.\n" + "Then the z_importkey can be used with this output\n" + "\nArguments:\n" + "1. \"zaddr\" (string, required) The zaddr for the private key\n" + "\nResult:\n" + "\"key\" (string) The private key\n" + "\nExamples:\n" + + HelpExampleCli("z_exportkey", "\"myaddress\"") + + HelpExampleCli("z_importkey", "\"mykey\"") + + HelpExampleRpc("z_exportkey", "\"myaddress\"") + ); + + LOCK2(cs_main, pwalletMain->cs_wallet); + + EnsureWalletIsUnlocked(); + + string strAddress = params[0].get_str(); + + CZCPaymentAddress address(strAddress); + auto addr = address.Get(); + if (!pwalletMain->HaveSpendingKey(addr)) + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet does not hold private zkey for this zaddr"); + + libzcash::SpendingKey k; + if (!pwalletMain->GetSpendingKey(addr, k)) + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet does not hold private zkey for this zaddr"); + + CZCSpendingKey spendingkey(k); + return spendingkey.ToString(); +} + diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index b9d588b9ac0..7315d7014b0 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2769,3 +2769,29 @@ Value zc_raw_keygen(const json_spirit::Array& params, bool fHelp) result.push_back(Pair("zcviewingkey", viewing_hex)); return result; } + + +Value z_getnewaddress(const Array& params, bool fHelp) +{ + if (!EnsureWalletIsAvailable(fHelp)) + return Value::null; + + if (fHelp || params.size() > 1) + throw runtime_error( + "z_getnewaddress\n" + "\nReturns a new zaddr for receiving payments.\n" + "\nArguments:\n" + "\nResult:\n" + "\"zcashaddress\" (string) The new zaddr\n" + "\nExamples:\n" + + HelpExampleCli("z_getnewaddress", "") + + HelpExampleRpc("z_getnewaddress", "") + ); + + LOCK2(cs_main, pwalletMain->cs_wallet); + + CZCPaymentAddress pubaddr = pwalletMain->GenerateNewZKey(); + std::string result = pubaddr.ToString(); + return result; +} + diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index f8215c939ea..d2878a40e90 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -25,6 +25,7 @@ #include using namespace std; +using namespace libzcash; /** * Settings @@ -70,6 +71,47 @@ const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const return &(it->second); } +// Generate a new spending key and return its public payment address +CZCPaymentAddress CWallet::GenerateNewZKey() +{ + AssertLockHeld(cs_wallet); // mapZKeyMetadata + auto k = SpendingKey::random(); + auto addr = k.address(); + + // Check for collision, even though it is unlikely to ever occur + if (CCryptoKeyStore::HaveSpendingKey(addr)) + throw std::runtime_error("CWallet::GenerateNewSpendingKey(): Collision detected"); + + // Create new metadata + int64_t nCreationTime = GetTime(); + mapZKeyMetadata[addr] = CKeyMetadata(nCreationTime); + + CZCPaymentAddress pubaddr(addr); + if (!AddZKey(k)) + throw std::runtime_error("CWallet::GenerateNewSpendingKey(): AddZKey failed"); + return pubaddr; +} + +// Add spending key to keystore and persist to disk +bool CWallet::AddZKey(const libzcash::SpendingKey &key) +{ + AssertLockHeld(cs_wallet); // mapZKeyMetadata + auto addr = key.address(); + + if (!CCryptoKeyStore::AddSpendingKey(key)) + return false; + + if (!fFileBacked) + return true; + + if (!IsCrypted()) { + return CWalletDB(strWalletFile).WriteZKey(addr, + key, + mapZKeyMetadata[addr]); + } + return true; +} + CPubKey CWallet::GenerateNewKey() { AssertLockHeld(cs_wallet); // mapKeyMetadata @@ -149,11 +191,23 @@ bool CWallet::LoadKeyMetadata(const CPubKey &pubkey, const CKeyMetadata &meta) return true; } +bool CWallet::LoadZKeyMetadata(const PaymentAddress &addr, const CKeyMetadata &meta) +{ + AssertLockHeld(cs_wallet); // mapZKeyMetadata + mapZKeyMetadata[addr] = meta; + return true; +} + bool CWallet::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector &vchCryptedSecret) { return CCryptoKeyStore::AddCryptedKey(vchPubKey, vchCryptedSecret); } +bool CWallet::LoadZKey(const libzcash::SpendingKey &key) +{ + return CCryptoKeyStore::AddSpendingKey(key); +} + bool CWallet::AddCScript(const CScript& redeemScript) { if (!CCryptoKeyStore::AddCScript(redeemScript)) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index f84f857f51a..0651c640444 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -18,6 +18,8 @@ #include "wallet/crypter.h" #include "wallet/wallet_ismine.h" #include "wallet/walletdb.h" +#include "zcash/Address.hpp" +#include "base58.h" #include #include @@ -486,6 +488,7 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface std::set setKeyPool; std::map mapKeyMetadata; + std::map mapZKeyMetadata; typedef std::map MasterKeyMap; MasterKeyMap mapMasterKeys; @@ -595,6 +598,19 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface void GetKeyBirthTimes(std::map &mapKeyBirth) const; + /** + * ZKeys + */ + //! Generates a new zaddr + CZCPaymentAddress GenerateNewZKey(); + //! Adds spending key to the store, and saves it to disk + bool AddZKey(const libzcash::SpendingKey &key); + //! Adds spending key to the store, without saving it to disk (used by LoadWallet) + bool LoadZKey(const libzcash::SpendingKey &key); + //! Load spending key metadata (used by LoadWallet) + bool LoadZKeyMetadata(const libzcash::PaymentAddress &addr, const CKeyMetadata &meta); + + /** * Increment the next transaction order id * @return next transaction order id diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 6f08ad681ea..2917445a5d6 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -110,6 +110,17 @@ bool CWalletDB::WriteMasterKey(unsigned int nID, const CMasterKey& kMasterKey) return Write(std::make_pair(std::string("mkey"), nID), kMasterKey, true); } +bool CWalletDB::WriteZKey(const libzcash::PaymentAddress& addr, const libzcash::SpendingKey& key, const CKeyMetadata &keyMeta) +{ + nWalletDBUpdated++; + + if (!Write(std::make_pair(std::string("zkeymeta"), addr), keyMeta)) + return false; + + // pair is: tuple_key("spendingkey", paymentaddress) --> secretkey + return Write(std::make_pair(std::string("zkey"), addr), key, false); +} + bool CWalletDB::WriteCScript(const uint160& hash, const CScript& redeemScript) { nWalletDBUpdated++; @@ -330,13 +341,15 @@ class CWalletScanState { unsigned int nKeys; unsigned int nCKeys; unsigned int nKeyMeta; + unsigned int nZKeys; + unsigned int nZKeyMeta; bool fIsEncrypted; bool fAnyUnordered; int nFileVersion; vector vWalletUpgrade; CWalletScanState() { - nKeys = nCKeys = nKeyMeta = 0; + nKeys = nCKeys = nKeyMeta = nZKeys = nZKeyMeta = 0; fIsEncrypted = false; fAnyUnordered = false; nFileVersion = 0; @@ -429,6 +442,23 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, // so set the wallet birthday to the beginning of time. pwallet->nTimeFirstKey = 1; } + else if (strType == "zkey") + { + libzcash::PaymentAddress addr; + ssKey >> addr; + libzcash::SpendingKey key; + ssValue >> key; + + if (!pwallet->LoadZKey(key)) + { + strErr = "Error reading wallet database: LoadZKey failed"; + return false; + } + + wss.nZKeys++; + + CZCPaymentAddress pubaddr(addr); + } else if (strType == "key" || strType == "wkey") { CPubKey vchPubKey; @@ -538,6 +568,18 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, (keyMeta.nCreateTime < pwallet->nTimeFirstKey)) pwallet->nTimeFirstKey = keyMeta.nCreateTime; } + else if (strType == "zkeymeta") + { + libzcash::PaymentAddress addr; + ssKey >> addr; + CKeyMetadata keyMeta; + ssValue >> keyMeta; + wss.nZKeyMeta++; + + pwallet->LoadZKeyMetadata(addr, keyMeta); + + // ignore earliest key creation time as taddr will exist before any zaddr + } else if (strType == "defaultkey") { ssValue >> pwallet->vchDefaultKey; @@ -685,6 +727,10 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) LogPrintf("Keys: %u plaintext, %u encrypted, %u w/ metadata, %u total\n", wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys); + // TODO: Keep track of encrypted ZKeys + LogPrintf("ZKeys: %u plaintext, -- encrypted, %u w/metadata, %u total\n", + wss.nZKeys, wss.nZKeyMeta, wss.nZKeys + 0); + // nTimeFirstKey is only reliable if all keys have metadata if ((wss.nKeys + wss.nCKeys) != wss.nKeyMeta) pwallet->nTimeFirstKey = 1; // 0 would be considered 'no value' diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index bc1a104b5be..d261c9644de 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -10,6 +10,7 @@ #include "wallet/db.h" #include "key.h" #include "keystore.h" +#include "zcash/Address.hpp" #include #include @@ -130,6 +131,9 @@ class CWalletDB : public CDB static bool Recover(CDBEnv& dbenv, const std::string& filename, bool fOnlyKeys); static bool Recover(CDBEnv& dbenv, const std::string& filename); + /// Write spending key to wallet database, where key is payment address and value is spending key. + bool WriteZKey(const libzcash::PaymentAddress& addr, const libzcash::SpendingKey& key, const CKeyMetadata &keyMeta); + private: CWalletDB(const CWalletDB&); void operator=(const CWalletDB&); From 62eea0d1e7c848f42a56634279fd7ca0331e11a1 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 10 Aug 2016 14:35:37 -0700 Subject: [PATCH 03/22] Add z_importwallet and z_exportwallet to handle keys for both taddr and zaddr. Restore behaviour of dumpwallet and importwallet to only handle taddr. --- src/rpcserver.cpp | 4 +- src/rpcserver.h | 2 + src/wallet/rpcdump.cpp | 123 ++++++++++++++++++++++++++++++----------- 3 files changed, 96 insertions(+), 33 deletions(-) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index afeb88d4721..f3ad60a04de 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -384,7 +384,9 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "zcsamplejoinsplit", &zc_sample_joinsplit, true }, { "wallet", "z_getnewaddress", &z_getnewaddress, true }, { "wallet", "z_exportkey", &z_exportkey, true }, - { "wallet", "z_importkey", &z_importkey, true } + { "wallet", "z_importkey", &z_importkey, true }, + { "wallet", "z_exportwallet", &z_exportwallet, true }, + { "wallet", "z_importwallet", &z_importwallet, true } #endif // ENABLE_WALLET }; diff --git a/src/rpcserver.h b/src/rpcserver.h index f34914a45d0..05f12fe1ec4 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -246,6 +246,8 @@ extern json_spirit::Value getblocksubsidy(const json_spirit::Array& params, bool extern json_spirit::Value z_exportkey(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp extern json_spirit::Value z_importkey(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp extern json_spirit::Value z_getnewaddress(const json_spirit::Array& params, bool fHelp); // in rpcwallet.cpp +extern json_spirit::Value z_exportwallet(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp +extern json_spirit::Value z_importwallet(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp // in rest.cpp extern bool HTTPReq_REST(AcceptedConnection *conn, diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index db5f36ed0e1..865f9b7574f 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -27,6 +27,10 @@ using namespace std; void EnsureWalletIsUnlocked(); bool EnsureWalletIsAvailable(bool avoidException); +Value _dumpwallet(const Array& params, bool fHelp, bool fDumpZKeys); +Value _importwallet(const Array& params, bool fHelp, bool fImportZKeys); + + std::string static EncodeDumpTime(int64_t nTime) { return DateTimeStrFormat("%Y-%m-%dT%H:%M:%SZ", nTime); } @@ -217,6 +221,29 @@ Value importaddress(const Array& params, bool fHelp) return Value::null; } +Value z_importwallet(const Array& params, bool fHelp) +{ + if (!EnsureWalletIsAvailable(fHelp)) + return Value::null; + + if (fHelp || params.size() != 1) + throw runtime_error( + "z_importwallet \"filename\"\n" + "\nImports taddr and zaddr keys from a wallet export file (see z_exportwallet).\n" + "\nArguments:\n" + "1. \"filename\" (string, required) The wallet file\n" + "\nExamples:\n" + "\nDump the wallet\n" + + HelpExampleCli("z_exportwallet", "\"test\"") + + "\nImport the wallet\n" + + HelpExampleCli("z_importwallet", "\"test\"") + + "\nImport using the json rpc call\n" + + HelpExampleRpc("z_importwallet", "\"test\"") + ); + + return _importwallet(params, fHelp, true); +} + Value importwallet(const Array& params, bool fHelp) { if (!EnsureWalletIsAvailable(fHelp)) @@ -237,6 +264,11 @@ Value importwallet(const Array& params, bool fHelp) + HelpExampleRpc("importwallet", "\"test\"") ); + return _importwallet(params, fHelp, false); +} + +Value _importwallet(const Array& params, bool fHelp, bool fImportZKeys) +{ LOCK2(cs_main, pwalletMain->cs_wallet); EnsureWalletIsUnlocked(); @@ -267,30 +299,31 @@ Value importwallet(const Array& params, bool fHelp) continue; // Let's see if the address is a valid Zcash spending key - try { - CZCSpendingKey spendingkey(vstr[0]); - libzcash::SpendingKey key = spendingkey.Get(); - libzcash::PaymentAddress addr = key.address(); - if (pwalletMain->HaveSpendingKey(addr)) { - LogPrintf("Skipping import of zaddr %s (key already present)\n", CZCPaymentAddress(addr).ToString()); + if (fImportZKeys) { + try { + CZCSpendingKey spendingkey(vstr[0]); + libzcash::SpendingKey key = spendingkey.Get(); + libzcash::PaymentAddress addr = key.address(); + if (pwalletMain->HaveSpendingKey(addr)) { + LogPrintf("Skipping import of zaddr %s (key already present)\n", CZCPaymentAddress(addr).ToString()); + continue; + } + int64_t nTime = DecodeDumpTime(vstr[1]); + LogPrintf("Importing zaddr %s...\n", CZCPaymentAddress(addr).ToString()); + if (!pwalletMain->AddZKey(key)) { + // Something went wrong + fGood = false; + continue; + } + // Successfully imported zaddr. Now import the metadata. + pwalletMain->mapZKeyMetadata[addr].nCreateTime = nTime; continue; } - int64_t nTime = DecodeDumpTime(vstr[1]); - LogPrintf("Importing zaddr %s...\n", CZCPaymentAddress(addr).ToString()); - if (!pwalletMain->AddZKey(key)) { - // Something went wrong - fGood = false; - continue; + catch (...) { + // Not a valid spending key, so carry on and see if it's a Bitcoin style address. } - // Successfully imported zaddr. Now import the metadata. - pwalletMain->mapZKeyMetadata[addr].nCreateTime = nTime; - continue; - } - catch (...) { - // Not a valid spending key, so carry on and see if it's a Bitcoin style address. } - CBitcoinSecret vchSecret; if (!vchSecret.SetString(vstr[0])) continue; @@ -385,11 +418,31 @@ Value dumpprivkey(const Array& params, bool fHelp) } -Value dumpwallet(const Array& params, bool fHelp) + +Value z_exportwallet(const Array& params, bool fHelp) { if (!EnsureWalletIsAvailable(fHelp)) return Value::null; + if (fHelp || params.size() != 1) + throw runtime_error( + "z_exportwallet \"filename\"\n" + "\nExports all wallet keys, for taddr and zaddr, in a human-readable format.\n" + "\nArguments:\n" + "1. \"filename\" (string, required) The filename\n" + "\nExamples:\n" + + HelpExampleCli("z_exportwallet", "\"test\"") + + HelpExampleRpc("z_exportwallet", "\"test\"") + ); + + return _dumpwallet(params, fHelp, true); +} + +Value dumpwallet(const Array& params, bool fHelp) +{ + if (!EnsureWalletIsAvailable(fHelp)) + return Value::null; + if (fHelp || params.size() != 1) throw runtime_error( "dumpwallet \"filename\"\n" @@ -401,6 +454,11 @@ Value dumpwallet(const Array& params, bool fHelp) + HelpExampleRpc("dumpwallet", "\"test\"") ); + return _dumpwallet(params, fHelp, false); +} + +Value _dumpwallet(const Array& params, bool fHelp, bool fDumpZKeys) +{ LOCK2(cs_main, pwalletMain->cs_wallet); EnsureWalletIsUnlocked(); @@ -446,20 +504,21 @@ Value dumpwallet(const Array& params, bool fHelp) } file << "\n"; - // dump the zkeys - std::set addresses; - pwalletMain->GetPaymentAddresses(addresses); - file << "\n"; - file << "# Zkeys\n"; - file << "\n"; - for (auto addr : addresses ) { - libzcash::SpendingKey key; - if (pwalletMain->GetSpendingKey(addr, key)) { - std::string strTime = EncodeDumpTime(pwalletMain->mapZKeyMetadata[addr].nCreateTime); - file << strprintf("%s %s # zaddr=%s\n", CZCSpendingKey(key).ToString(), strTime, CZCPaymentAddress(addr).ToString()); + if (fDumpZKeys) { + std::set addresses; + pwalletMain->GetPaymentAddresses(addresses); + file << "\n"; + file << "# Zkeys\n"; + file << "\n"; + for (auto addr : addresses ) { + libzcash::SpendingKey key; + if (pwalletMain->GetSpendingKey(addr, key)) { + std::string strTime = EncodeDumpTime(pwalletMain->mapZKeyMetadata[addr].nCreateTime); + file << strprintf("%s %s # zaddr=%s\n", CZCSpendingKey(key).ToString(), strTime, CZCPaymentAddress(addr).ToString()); + } } + file << "\n"; } - file << "\n"; file << "# End of dump\n"; file.close(); From 84ce3a51f5a54dd5254123ec7932fd0ce649c4ae Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 10 Aug 2016 15:02:00 -0700 Subject: [PATCH 04/22] Implemented z_listaddresses to return all the zaddr in the wallet. --- src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/wallet/rpcwallet.cpp | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index f3ad60a04de..4d38a579f4c 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -383,6 +383,7 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "zcrawreceive", &zc_raw_receive, true }, { "wallet", "zcsamplejoinsplit", &zc_sample_joinsplit, true }, { "wallet", "z_getnewaddress", &z_getnewaddress, true }, + { "wallet", "z_listaddresses", &z_listaddresses, true }, { "wallet", "z_exportkey", &z_exportkey, true }, { "wallet", "z_importkey", &z_importkey, true }, { "wallet", "z_exportwallet", &z_exportwallet, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 05f12fe1ec4..0e8b60b7b52 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -246,6 +246,7 @@ extern json_spirit::Value getblocksubsidy(const json_spirit::Array& params, bool extern json_spirit::Value z_exportkey(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp extern json_spirit::Value z_importkey(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp extern json_spirit::Value z_getnewaddress(const json_spirit::Array& params, bool fHelp); // in rpcwallet.cpp +extern json_spirit::Value z_listaddresses(const json_spirit::Array& params, bool fHelp); // in rpcwallet.cpp extern json_spirit::Value z_exportwallet(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp extern json_spirit::Value z_importwallet(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 7315d7014b0..e37f56b5018 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2795,3 +2795,35 @@ Value z_getnewaddress(const Array& params, bool fHelp) return result; } + +Value z_listaddresses(const Array& params, bool fHelp) +{ + if (!EnsureWalletIsAvailable(fHelp)) + return Value::null; + + if (fHelp || params.size() > 1) + throw runtime_error( + "z_listaddresses\n" + "\nReturns the list of zaddr belonging to the wallet.\n" + "\nArguments:\n" + "\nResult:\n" + "[ (json array of string)\n" + " \"zaddr\" (string) a zaddr belonging to the wallet\n" + " ,...\n" + "]\n" + "\nExamples:\n" + + HelpExampleCli("z_listaddresses", "") + + HelpExampleRpc("z_listaddresses", "") + ); + + LOCK2(cs_main, pwalletMain->cs_wallet); + + Array ret; + std::set addresses; + pwalletMain->GetPaymentAddresses(addresses); + for (auto addr : addresses ) { + ret.push_back(CZCPaymentAddress(addr).ToString()); + } + return ret; +} + From 65c02f9016fc9110175ce13a7ad705ce50f2d5d1 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Aug 2016 10:48:16 -0700 Subject: [PATCH 05/22] Add gtest to cover new methods in: CWallet - GenerateNewZKey() - AddZKey() - LoadZKey() - LoadZKeyMetadata() CWalletDB - WriteZKey() --- src/Makefile.gtest.include | 1 + src/gtest/test_wallet_zkeys.cpp | 140 ++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 src/gtest/test_wallet_zkeys.cpp diff --git a/src/Makefile.gtest.include b/src/Makefile.gtest.include index 99fdf1e754d..2f5fc596dbf 100644 --- a/src/Makefile.gtest.include +++ b/src/Makefile.gtest.include @@ -13,6 +13,7 @@ zcash_gtest_SOURCES = \ gtest/test_merkletree.cpp \ gtest/test_circuit.cpp \ gtest/test_txid.cpp \ + gtest/test_wallet_zkeys.cpp \ gtest/test_libzcash_utils.cpp zcash_gtest_CPPFLAGS = -DMULTICORE -fopenmp -DBINARY_OUTPUT -DCURVE_ALT_BN128 -DSTATIC diff --git a/src/gtest/test_wallet_zkeys.cpp b/src/gtest/test_wallet_zkeys.cpp new file mode 100644 index 00000000000..071d89fadc2 --- /dev/null +++ b/src/gtest/test_wallet_zkeys.cpp @@ -0,0 +1,140 @@ +#include + +#include "zcash/Address.hpp" +#include "wallet/wallet.h" +#include "wallet/walletdb.h" +#include "util.h" + +#include + +/** + * This test covers methods on CWallet + * GenerateNewZKey() + * AddZKey() + * LoadZKey() + * LoadZKeyMetadata() + */ +TEST(wallet_zkeys_tests, store_and_load_zkeys) { + SelectParams(CBaseChainParams::MAIN); + + CWallet wallet; + + // wallet should by empty + std::set addrs; + wallet.GetPaymentAddresses(addrs); + ASSERT_EQ(0, addrs.size()); + + // wallet should have one key + CZCPaymentAddress paymentAddress = wallet.GenerateNewZKey(); + wallet.GetPaymentAddresses(addrs); + ASSERT_EQ(1, addrs.size()); + + // verify wallet has spending key for the address + auto addr = paymentAddress.Get(); + ASSERT_TRUE(wallet.HaveSpendingKey(addr)); + + // manually add new spending key to wallet + auto sk = libzcash::SpendingKey::random(); + ASSERT_TRUE(wallet.AddZKey(sk)); + + // verify wallet did add it + addr = sk.address(); + ASSERT_TRUE(wallet.HaveSpendingKey(addr)); + + // verify spending key stored correctly + libzcash::SpendingKey keyOut; + wallet.GetSpendingKey(addr, keyOut); + ASSERT_EQ(sk, keyOut); + + // verify there are two keys + wallet.GetPaymentAddresses(addrs); + ASSERT_EQ(2, addrs.size()); + ASSERT_EQ(1, addrs.count(addr)); + + // Load a third key into the wallet + sk = libzcash::SpendingKey::random(); + ASSERT_TRUE(wallet.LoadZKey(sk)); + + // attach metadata to this third key + addr = sk.address(); + int64_t now = GetTime(); + CKeyMetadata meta(now); + ASSERT_TRUE(wallet.LoadZKeyMetadata(addr, meta)); + + // check metadata is the same + CKeyMetadata m= wallet.mapZKeyMetadata[addr]; + ASSERT_EQ(m.nCreateTime, now); +} + +/** + * This test covers methods on CWalletDB + * WriteZKey() + */ +TEST(wallet_zkeys_tests, write_zkey_direct_to_db) { + SelectParams(CBaseChainParams::TESTNET); + + // Get temporary and unique path for file. + // Note: / operator to append paths + boost::filesystem::path temp = boost::filesystem::temp_directory_path() / + boost::filesystem::unique_path(); + const std::string path = temp.native(); + + bool fFirstRun; + CWallet wallet(path); + ASSERT_EQ(DB_LOAD_OK, wallet.LoadWallet(fFirstRun)); + + // No default CPubKey set + ASSERT_TRUE(fFirstRun); + + // wallet should by empty + std::set addrs; + wallet.GetPaymentAddresses(addrs); + ASSERT_EQ(0, addrs.size()); + + // Add random key to the wallet + auto paymentAddress = wallet.GenerateNewZKey(); + + // wallet should have one key + wallet.GetPaymentAddresses(addrs); + ASSERT_EQ(1, addrs.size()); + + // create random key and add it to database directly, bypassing wallet + auto sk = libzcash::SpendingKey::random(); + auto addr = sk.address(); + int64_t now = GetTime(); + CKeyMetadata meta(now); + CWalletDB db(path); + db.WriteZKey(addr, sk, meta); + + // wallet should not be aware of key + ASSERT_FALSE(wallet.HaveSpendingKey(addr)); + + // wallet sees one key + wallet.GetPaymentAddresses(addrs); + ASSERT_EQ(1, addrs.size()); + + // wallet should have default metadata for addr with null createtime + CKeyMetadata m = wallet.mapZKeyMetadata[addr]; + ASSERT_EQ(m.nCreateTime, 0); + ASSERT_NE(m.nCreateTime, now); + + // load the wallet again + ASSERT_EQ(DB_LOAD_OK, wallet.LoadWallet(fFirstRun)); + + // wallet can now see the spending key + ASSERT_TRUE(wallet.HaveSpendingKey(addr)); + + // check key is the same + libzcash::SpendingKey keyOut; + wallet.GetSpendingKey(addr, keyOut); + ASSERT_EQ(sk, keyOut); + + // wallet should have two keys + wallet.GetPaymentAddresses(addrs); + ASSERT_EQ(2, addrs.size()); + + // check metadata is now the same + m = wallet.mapZKeyMetadata[addr]; + ASSERT_EQ(m.nCreateTime, now); +} + From 9155cc6e42d1cf0ede51845efcffd54472f69e89 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 11 Aug 2016 11:37:00 -0700 Subject: [PATCH 06/22] Don't mark wallet as dirty if key already exists. Fix incorrect method name used in error message. --- src/wallet/rpcdump.cpp | 4 ++-- src/wallet/wallet.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 865f9b7574f..35c372b9d7d 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -565,12 +565,12 @@ Value z_importkey(const Array& params, bool fHelp) auto addr = key.address(); { - pwalletMain->MarkDirty(); - // Don't throw error in case a key is already there if (pwalletMain->HaveSpendingKey(addr)) return Value::null; + pwalletMain->MarkDirty(); + if (!pwalletMain-> AddZKey(key)) throw JSONRPCError(RPC_WALLET_ERROR, "Error adding spending key to wallet"); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index d2878a40e90..0f6eeff49fd 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -80,7 +80,7 @@ CZCPaymentAddress CWallet::GenerateNewZKey() // Check for collision, even though it is unlikely to ever occur if (CCryptoKeyStore::HaveSpendingKey(addr)) - throw std::runtime_error("CWallet::GenerateNewSpendingKey(): Collision detected"); + throw std::runtime_error("CWallet::GenerateNewZKey(): Collision detected"); // Create new metadata int64_t nCreationTime = GetTime(); @@ -88,7 +88,7 @@ CZCPaymentAddress CWallet::GenerateNewZKey() CZCPaymentAddress pubaddr(addr); if (!AddZKey(k)) - throw std::runtime_error("CWallet::GenerateNewSpendingKey(): AddZKey failed"); + throw std::runtime_error("CWallet::GenerateNewZKey(): AddZKey failed"); return pubaddr; } From e57e5e34974a053b3de2114ab445e8993f7d3da1 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 12 Aug 2016 17:44:03 -0700 Subject: [PATCH 07/22] Added wallet rpc tests to cover: z_importwallet, z_exportwallet z_importkey, z_exportkey z_listaddresses --- src/test/rpc_wallet_tests.cpp | 208 +++++++++++++++++++++++++++++++++- 1 file changed, 207 insertions(+), 1 deletion(-) diff --git a/src/test/rpc_wallet_tests.cpp b/src/test/rpc_wallet_tests.cpp index 4d5e92cbd40..9644964d174 100644 --- a/src/test/rpc_wallet_tests.cpp +++ b/src/test/rpc_wallet_tests.cpp @@ -11,8 +11,14 @@ #include "test/test_bitcoin.h" +#include "zcash/Address.hpp" + +#include +#include + #include #include +#include using namespace std; using namespace json_spirit; @@ -218,4 +224,204 @@ BOOST_AUTO_TEST_CASE(rpc_wallet) BOOST_CHECK(CBitcoinAddress(arr[0].get_str()).Get() == demoAddress.Get()); } -BOOST_AUTO_TEST_SUITE_END() +/* + * This test covers RPC command z_exportwallet + */ +BOOST_AUTO_TEST_CASE(rpc_wallet_z_exportwallet) +{ + LOCK2(cs_main, pwalletMain->cs_wallet); + + // wallet should by empty + std::set addrs; + pwalletMain->GetPaymentAddresses(addrs); + BOOST_CHECK(addrs.size()==0); + + // wallet should have one key + CZCPaymentAddress paymentAddress = pwalletMain->GenerateNewZKey(); + pwalletMain->GetPaymentAddresses(addrs); + BOOST_CHECK(addrs.size()==1); + + BOOST_CHECK_THROW(CallRPC("z_exportwallet"), runtime_error); + + + boost::filesystem::path temp = boost::filesystem::temp_directory_path() / + boost::filesystem::unique_path(); + const std::string path = temp.native(); + + BOOST_CHECK_NO_THROW(CallRPC(string("z_exportwallet ") + path)); + + auto addr = paymentAddress.Get(); + libzcash::SpendingKey key; + BOOST_CHECK(pwalletMain->GetSpendingKey(addr, key)); + + std::string s1 = paymentAddress.ToString(); + std::string s2 = CZCSpendingKey(key).ToString(); + + // There's no way to really delete a private key so we will read in the + // exported wallet file and search for the spending key and payment address. + + EnsureWalletIsUnlocked(); + + ifstream file; + file.open(path.c_str(), std::ios::in | std::ios::ate); + BOOST_CHECK(file.is_open()); + bool fVerified = false; + int64_t nFilesize = std::max((int64_t)1, (int64_t)file.tellg()); + file.seekg(0, file.beg); + while (file.good()) { + std::string line; + std::getline(file, line); + if (line.empty() || line[0] == '#') + continue; + if (line.find(s1) != std::string::npos && line.find(s2) != std::string::npos) { + fVerified = true; + break; + } + } + BOOST_CHECK(fVerified); +} + + +/* + * This test covers RPC command z_importwallet + */ +BOOST_AUTO_TEST_CASE(rpc_wallet_z_importwallet) +{ + LOCK2(cs_main, pwalletMain->cs_wallet); + + // error if no args + BOOST_CHECK_THROW(CallRPC("z_importwallet"), runtime_error); + + // create a random key locally + auto testSpendingKey = libzcash::SpendingKey::random(); + auto testPaymentAddress = testSpendingKey.address(); + std::string testAddr = CZCPaymentAddress(testPaymentAddress).ToString(); + std::string testKey = CZCSpendingKey(testSpendingKey).ToString(); + +// std::cout << "testAddr = " << testAddr << std::endl; +// std::cout << "testKey = " << testKey << std::endl; + + // create test data using the random key + std::string format_str = "# Wallet dump created by Zcash v0.11.2.0.z8-9155cc6-dirty (2016-08-11 11:37:00 -0700)\n" + "# * Created on 2016-08-12T21:55:36Z\n" + "# * Best block at time of backup was 0 (0de0a3851fef2d433b9b4f51d4342bdd24c5ddd793eb8fba57189f07e9235d52),\n" + "# mined on 2009-01-03T18:15:05Z\n" + "\n" + "# Zkeys\n" + "\n" + "%s 2016-08-12T21:55:36Z # zaddr=%s\n" + "\n" + "\n# End of dump"; + + boost::format formatobject(format_str); + std::string testWalletDump = (formatobject % testKey % testAddr).str(); + +// std::cout << "testWalletDump =\n" << testWalletDump << std::endl; + + // write test data to file + boost::filesystem::path temp = boost::filesystem::temp_directory_path() / + boost::filesystem::unique_path(); + const std::string path = temp.native(); + std::ofstream file(path); + file << testWalletDump; + file << std::flush; + + // wallet should currently be empty + std::set addrs; + pwalletMain->GetPaymentAddresses(addrs); + BOOST_CHECK(addrs.size()==0); + + // import test data from file into wallet + BOOST_CHECK_NO_THROW(CallRPC(string("z_importwallet ") + path)); + + // wallet should now have one zkey + pwalletMain->GetPaymentAddresses(addrs); + BOOST_CHECK(addrs.size()==1); + + // check that we have the spending key for the address + CZCPaymentAddress address(testAddr); + auto addr = address.Get(); + BOOST_CHECK(pwalletMain->HaveSpendingKey(addr)); + + // Verify the spending keys is the same as the test data + libzcash::SpendingKey k; + BOOST_CHECK(pwalletMain->GetSpendingKey(addr, k)); + CZCSpendingKey spendingkey(k); + BOOST_CHECK_EQUAL(testKey, spendingkey.ToString()); + +// std::cout << "address = " << address.ToString() << std::endl; +// std::cout << "spendingkey = " << spendingkey.ToString() << std::endl; +} + + +/* + * This test covers RPC command z_listaddresses, z_importkey, z_exportkey + */ +BOOST_AUTO_TEST_CASE(rpc_wallet_z_importexport) +{ + LOCK2(cs_main, pwalletMain->cs_wallet); + Value retValue; + int n1 = 1000; // number of times to import/export + int n2 = 1000; // number of addresses to create and list + + // error if no args + BOOST_CHECK_THROW(CallRPC("z_importkey"), runtime_error); + BOOST_CHECK_THROW(CallRPC("z_exportkey"), runtime_error); + + // wallet should currently be empty + std::set addrs; + pwalletMain->GetPaymentAddresses(addrs); + BOOST_CHECK(addrs.size()==0); + + // verify import and export key + for (int i = 0; i < n1; i++) { + // create a random key locally + auto testSpendingKey = libzcash::SpendingKey::random(); + auto testPaymentAddress = testSpendingKey.address(); + std::string testAddr = CZCPaymentAddress(testPaymentAddress).ToString(); + std::string testKey = CZCSpendingKey(testSpendingKey).ToString(); + BOOST_CHECK_NO_THROW(CallRPC(string("z_importkey ") + testKey)); + BOOST_CHECK_NO_THROW(retValue = CallRPC(string("z_exportkey ") + testAddr)); + BOOST_CHECK_EQUAL(retValue.get_str(), testKey); + } + + // Verify we can list the keys imported + BOOST_CHECK_NO_THROW(retValue = CallRPC("z_listaddresses")); + Array arr = retValue.get_array(); + BOOST_CHECK(arr.size() == n1); + + // Put addresses into a set + std::unordered_set myaddrs; + for (Value element : arr) { + myaddrs.insert(element.get_str()); + } + + // Make new addresses for the set + for (int i=0; iGenerateNewZKey()).ToString()); + } + + // Verify number of addresses stored in wallet is n1+n2 + int numAddrs = myaddrs.size(); + BOOST_CHECK(numAddrs == n1+n2); + pwalletMain->GetPaymentAddresses(addrs); + BOOST_CHECK(addrs.size()==numAddrs); + + // Ask wallet to list addresses + BOOST_CHECK_NO_THROW(retValue = CallRPC("z_listaddresses")); + arr = retValue.get_array(); + BOOST_CHECK(arr.size() == numAddrs); + + // Create a set form them + std::unordered_set listaddrs; + for (Value element : arr) { + listaddrs.insert(element.get_str()); + } + + // Verify the two sets of addresses are the same + BOOST_CHECK(listaddrs.size() == numAddrs); + BOOST_CHECK(myaddrs == listaddrs); +} + + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file From 4df39992da99da69c477b5e99d7b8bc19863647d Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 12 Aug 2016 14:24:29 +1200 Subject: [PATCH 08/22] Merge AddSpendingKeyPaymentAddress into AddSpendingKey to simplify API --- src/keystore.cpp | 8 ++------ src/keystore.h | 5 ++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/keystore.cpp b/src/keystore.cpp index 376654e7f5b..ef1878a9bf6 100644 --- a/src/keystore.cpp +++ b/src/keystore.cpp @@ -23,10 +23,6 @@ bool CKeyStore::AddKey(const CKey &key) { return AddKeyPubKey(key, key.GetPubKey()); } -bool CKeyStore::AddSpendingKey(const libzcash::SpendingKey &key) { - return AddSpendingKeyPaymentAddress(key, key.address()); -} - bool CBasicKeyStore::AddKeyPubKey(const CKey& key, const CPubKey &pubkey) { LOCK(cs_KeyStore); @@ -88,9 +84,9 @@ bool CBasicKeyStore::HaveWatchOnly() const return (!setWatchOnly.empty()); } -bool CBasicKeyStore::AddSpendingKeyPaymentAddress(const libzcash::SpendingKey& key, const libzcash::PaymentAddress &address) +bool CBasicKeyStore::AddSpendingKey(const libzcash::SpendingKey &sk) { LOCK(cs_KeyStore); - mapSpendingKeys[address] = key; + mapSpendingKeys[sk.address()] = sk; return true; } diff --git a/src/keystore.h b/src/keystore.h index bbd04f235c8..4beba84feaf 100644 --- a/src/keystore.h +++ b/src/keystore.h @@ -47,8 +47,7 @@ class CKeyStore virtual bool HaveWatchOnly() const =0; //! Add a spending key to the store. - virtual bool AddSpendingKeyPaymentAddress(const libzcash::SpendingKey &key, const libzcash::PaymentAddress &address) =0; - virtual bool AddSpendingKey(const libzcash::SpendingKey &key); + virtual bool AddSpendingKey(const libzcash::SpendingKey &sk) =0; //! Check whether a spending key corresponding to a given payment address is present in the store. virtual bool HaveSpendingKey(const libzcash::PaymentAddress &address) const =0; @@ -116,7 +115,7 @@ class CBasicKeyStore : public CKeyStore virtual bool HaveWatchOnly(const CScript &dest) const; virtual bool HaveWatchOnly() const; - bool AddSpendingKeyPaymentAddress(const libzcash::SpendingKey &key, const libzcash::PaymentAddress &address); + bool AddSpendingKey(const libzcash::SpendingKey &sk); bool HaveSpendingKey(const libzcash::PaymentAddress &address) const { bool result; From 8cec3de25b539ba5ad90cea898d77760ad343408 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 12 Aug 2016 14:37:17 +1200 Subject: [PATCH 09/22] Consistent parameter naming --- src/gtest/test_keystore.cpp | 6 +++--- src/keystore.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/gtest/test_keystore.cpp b/src/gtest/test_keystore.cpp index 987bdd13acc..8baf02cab58 100644 --- a/src/gtest/test_keystore.cpp +++ b/src/gtest/test_keystore.cpp @@ -16,9 +16,9 @@ TEST(keystore_tests, store_and_retrieve_spending_key) { auto addr = sk.address(); ASSERT_TRUE(keyStore.HaveSpendingKey(addr)); - libzcash::SpendingKey keyOut; - keyStore.GetSpendingKey(addr, keyOut); - ASSERT_EQ(sk, keyOut); + libzcash::SpendingKey skOut; + keyStore.GetSpendingKey(addr, skOut); + ASSERT_EQ(sk, skOut); keyStore.GetPaymentAddresses(addrs); ASSERT_EQ(1, addrs.size()); diff --git a/src/keystore.h b/src/keystore.h index 4beba84feaf..927ebe221d4 100644 --- a/src/keystore.h +++ b/src/keystore.h @@ -51,7 +51,7 @@ class CKeyStore //! Check whether a spending key corresponding to a given payment address is present in the store. virtual bool HaveSpendingKey(const libzcash::PaymentAddress &address) const =0; - virtual bool GetSpendingKey(const libzcash::PaymentAddress &address, libzcash::SpendingKey& keyOut) const =0; + virtual bool GetSpendingKey(const libzcash::PaymentAddress &address, libzcash::SpendingKey& skOut) const =0; virtual void GetPaymentAddresses(std::set &setAddress) const =0; }; @@ -125,14 +125,14 @@ class CBasicKeyStore : public CKeyStore } return result; } - bool GetSpendingKey(const libzcash::PaymentAddress &address, libzcash::SpendingKey &keyOut) const + bool GetSpendingKey(const libzcash::PaymentAddress &address, libzcash::SpendingKey &skOut) const { { LOCK(cs_KeyStore); SpendingKeyMap::const_iterator mi = mapSpendingKeys.find(address); if (mi != mapSpendingKeys.end()) { - keyOut = mi->second; + skOut = mi->second; return true; } } From d78026d3a3cd3592b2870703e5db3b35ae81c065 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 16 Aug 2016 22:17:33 +1200 Subject: [PATCH 10/22] Add separate lock for SpendingKey key store operations --- src/keystore.cpp | 2 +- src/keystore.h | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/keystore.cpp b/src/keystore.cpp index ef1878a9bf6..7240cd74733 100644 --- a/src/keystore.cpp +++ b/src/keystore.cpp @@ -86,7 +86,7 @@ bool CBasicKeyStore::HaveWatchOnly() const bool CBasicKeyStore::AddSpendingKey(const libzcash::SpendingKey &sk) { - LOCK(cs_KeyStore); + LOCK(cs_SpendingKeyStore); mapSpendingKeys[sk.address()] = sk; return true; } diff --git a/src/keystore.h b/src/keystore.h index 927ebe221d4..987f32070f7 100644 --- a/src/keystore.h +++ b/src/keystore.h @@ -21,6 +21,7 @@ class CKeyStore { protected: mutable CCriticalSection cs_KeyStore; + mutable CCriticalSection cs_SpendingKeyStore; public: virtual ~CKeyStore() {} @@ -120,7 +121,7 @@ class CBasicKeyStore : public CKeyStore { bool result; { - LOCK(cs_KeyStore); + LOCK(cs_SpendingKeyStore); result = (mapSpendingKeys.count(address) > 0); } return result; @@ -128,7 +129,7 @@ class CBasicKeyStore : public CKeyStore bool GetSpendingKey(const libzcash::PaymentAddress &address, libzcash::SpendingKey &skOut) const { { - LOCK(cs_KeyStore); + LOCK(cs_SpendingKeyStore); SpendingKeyMap::const_iterator mi = mapSpendingKeys.find(address); if (mi != mapSpendingKeys.end()) { @@ -142,7 +143,7 @@ class CBasicKeyStore : public CKeyStore { setAddress.clear(); { - LOCK(cs_KeyStore); + LOCK(cs_SpendingKeyStore); SpendingKeyMap::const_iterator mi = mapSpendingKeys.begin(); while (mi != mapSpendingKeys.end()) { From 9a3a4271685feea367823c5288f37cc511168cc1 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 16 Aug 2016 10:33:04 -0700 Subject: [PATCH 11/22] Add async RPC queue and operation classes. Add z_getoperationstatus RPC command. Add z_sendmany RPC command (dummy implementation, does not send actual coins). --- src/Makefile.am | 6 + src/asyncrpcoperation.cpp | 147 +++++++++++++++++ src/asyncrpcoperation.h | 148 +++++++++++++++++ src/asyncrpcqueue.cpp | 147 +++++++++++++++++ src/asyncrpcqueue.h | 61 +++++++ src/rpcclient.cpp | 2 + src/rpcserver.cpp | 23 +++ src/rpcserver.h | 9 + src/wallet/asyncrpcoperation_sendmany.cpp | 77 +++++++++ src/wallet/asyncrpcoperation_sendmany.h | 32 ++++ src/wallet/rpcwallet.cpp | 191 ++++++++++++++++++++++ 11 files changed, 843 insertions(+) create mode 100644 src/asyncrpcoperation.cpp create mode 100644 src/asyncrpcoperation.h create mode 100644 src/asyncrpcqueue.cpp create mode 100644 src/asyncrpcqueue.h create mode 100644 src/wallet/asyncrpcoperation_sendmany.cpp create mode 100644 src/wallet/asyncrpcoperation_sendmany.h diff --git a/src/Makefile.am b/src/Makefile.am index 7c843cba805..98b078a9049 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -86,6 +86,8 @@ BITCOIN_CORE_H = \ alert.h \ amount.h \ arith_uint256.h \ + asyncrpcoperation.h \ + asyncrpcqueue.h \ base58.h \ bloom.h \ chain.h \ @@ -162,6 +164,7 @@ BITCOIN_CORE_H = \ utiltime.h \ validationinterface.h \ version.h \ + wallet/asyncrpcoperation_sendmany.h \ wallet/crypter.h \ wallet/db.h \ wallet/wallet.h \ @@ -191,6 +194,8 @@ libbitcoin_server_a_SOURCES = \ sendalert.cpp \ addrman.cpp \ alert.cpp \ + asyncrpcoperation.cpp \ + asyncrpcqueue.cpp \ bloom.cpp \ chain.cpp \ checkpoints.cpp \ @@ -224,6 +229,7 @@ libbitcoin_server_a_SOURCES = \ libbitcoin_wallet_a_CPPFLAGS = $(BITCOIN_INCLUDES) libbitcoin_wallet_a_SOURCES = \ zcbenchmarks.cpp \ + wallet/asyncrpcoperation_sendmany.cpp \ wallet/crypter.cpp \ wallet/db.cpp \ wallet/rpcdump.cpp \ diff --git a/src/asyncrpcoperation.cpp b/src/asyncrpcoperation.cpp new file mode 100644 index 00000000000..debe2adc988 --- /dev/null +++ b/src/asyncrpcoperation.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2016 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "asyncrpcoperation.h" + +#include +#include +#include + +#include +#include +#include + +using namespace std; +using namespace json_spirit; + +static boost::uuids::random_generator uuidgen; + +std::map OperationStatusMap = { + {OperationStatus::READY, "queued"}, + {OperationStatus::EXECUTING, "executing"}, + {OperationStatus::CANCELLED, "cancelled"}, + {OperationStatus::FAILED, "failed"}, + {OperationStatus::SUCCESS, "success"} +}; + +AsyncRPCOperation::AsyncRPCOperation() : errorCode(0), errorMessage() { + // Set a unique reference for each operation + boost::uuids::uuid uuid = uuidgen(); + std::string s = boost::uuids::to_string(uuid); + setId(s); + + setState(OperationStatus::READY); + creationTime = (int64_t)time(NULL); +} + +AsyncRPCOperation::AsyncRPCOperation(const AsyncRPCOperation& o) : id(o.id), creationTime(o.creationTime), state(o.state.load()) +{ +} + + +AsyncRPCOperation& AsyncRPCOperation::operator=( const AsyncRPCOperation& other ) { + this->id = other.getId(); + this->creationTime = other.creationTime; + this->state.store(other.state.load()); + return *this; +} + + +AsyncRPCOperation::~AsyncRPCOperation() { +} + +void AsyncRPCOperation::cancel() { + if (isReady()) + setState(OperationStatus::CANCELLED); +} + + +void AsyncRPCOperation::startExecutionClock() { + startTime = std::chrono::system_clock::now(); +} + +void AsyncRPCOperation::stopExecutionClock() { + endTime = std::chrono::system_clock::now(); +} + + +// Implement this method in any subclass. +// This is just an example implementation. + + +void AsyncRPCOperation::main() { + if (isCancelled()) + return; + + setState(OperationStatus::EXECUTING); + + // + // Do some work here... + // + + startExecutionClock(); + + //std::this_thread::sleep_for(std::chrono::milliseconds(10000)); + + stopExecutionClock(); + + + // If there was an error... +// setErrorCode(123); +// setErrorMessage("Murphy's law"); +// setState(OperationStatus::FAILED); + + + // Otherwise + Value v("We have a result!"); + setResult(v); + setState(OperationStatus::SUCCESS); +} + +Value AsyncRPCOperation::getError() const { + if (!isFailed()) + return Value::null; + + Object error; + error.push_back(Pair("code", this->errorCode)); + error.push_back(Pair("message", this->errorMessage)); + return Value(error); +} + +Value AsyncRPCOperation::getResult() const { + if (!isSuccess()) + return Value::null; + + return this->resultValue; +} + + +/* + * Returns a status Value object. + * If the operation has failed, it will include an error object. + * If the operation has succeeded, it will include the result value. + */ +Value AsyncRPCOperation::getStatus() const { + OperationStatus status = this->getState(); + Object obj; + obj.push_back(Pair("id", this->getId())); + obj.push_back(Pair("status", OperationStatusMap[status])); + obj.push_back(Pair("creation_time", this->creationTime)); + // creation, exec time, duration, exec end, etc. + Value err = this->getError(); + if (!err.is_null()) { + obj.push_back(Pair("error", err.get_obj())); + } + Value result = this->getResult(); + if (!result.is_null()) { + obj.push_back(Pair("result", result)); + + // Include execution time for successful operation + std::chrono::duration elapsed_seconds = endTime - startTime; + obj.push_back(Pair("execution_secs", elapsed_seconds.count())); + + } + return Value(obj); +} + diff --git a/src/asyncrpcoperation.h b/src/asyncrpcoperation.h new file mode 100644 index 00000000000..f30c174d61e --- /dev/null +++ b/src/asyncrpcoperation.h @@ -0,0 +1,148 @@ +// Copyright (c) 2016 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + + +#ifndef ASYNCRPCOPERATION_H +#define ASYNCRPCOPERATION_H + +#include +#include +#include +#include + +#include "json/json_spirit_value.h" +#include "json/json_spirit_utils.h" +#include "json/json_spirit_reader_template.h" +#include "json/json_spirit_writer_template.h" + +using namespace std; +using namespace json_spirit; + +/** + * AsyncRPCOperations are given to the AsyncRPCQueue for processing. + * + * How to subclass: + * Implement the main() method, this is where work is performed. + * Update the operation status as work is underway and completes. + */ + +typedef std::string AsyncRPCOperationId; + +typedef enum class operationStateEnum { + READY = 0, + EXECUTING, + CANCELLED, + FAILED, + SUCCESS +} OperationStatus; + +class AsyncRPCOperation { +public: + AsyncRPCOperation(); + + // Todo: keep or delete copy constructors and assignment? + AsyncRPCOperation(const AsyncRPCOperation& orig); + AsyncRPCOperation& operator=( const AsyncRPCOperation& other ); + + virtual ~AsyncRPCOperation(); + + // Implement this method in your subclass. + virtual void main(); + + void cancel(); + + // Getters and setters + + OperationStatus getState() const { + return state.load(); + } + + AsyncRPCOperationId getId() const { + return id; + } + + int64_t getCreationTime() const { + return creationTime; + } + + Value getStatus() const; + + Value getError() const; + + Value getResult() const; + + int getErrorCode() const { + return errorCode; + } + + std::string getErrorMessage() const { + return errorMessage; + } + + bool isCancelled() const { + return OperationStatus::CANCELLED==getState(); + } + + bool isExecuting() const { + return OperationStatus::EXECUTING==getState(); + } + + bool isReady() const { + return OperationStatus::READY==getState(); + } + + bool isFailed() const { + return OperationStatus::FAILED==getState(); + } + + bool isSuccess() const { + return OperationStatus::SUCCESS==getState(); + } + +protected: + + Value resultValue; + int errorCode; + std::string errorMessage; + std::atomic state; + std::chrono::time_point startTime, endTime; + + void startExecutionClock(); + void stopExecutionClock(); + + void setState(OperationStatus state) { + this->state.store(state); + } + + void setErrorCode(int errorCode) { + this->errorCode = errorCode; + } + + void setErrorMessage(std::string errorMessage) { + this->errorMessage = errorMessage; + } + + void setResult(Value v) { + this->resultValue = v; + } + +private: + + // Todo: Private for now. If copying an operation is possible, should it + // receive a new id and a new creation time? + void setId(AsyncRPCOperationId id) { + this->id = id; + } + + // Todo: Ditto above. + void setCreationTime(int64_t creationTime) { + this->creationTime = creationTime; + } + + AsyncRPCOperationId id; + int64_t creationTime; +}; + +#endif /* ASYNCRPCOPERATION_H */ + diff --git a/src/asyncrpcqueue.cpp b/src/asyncrpcqueue.cpp new file mode 100644 index 00000000000..679f861c58c --- /dev/null +++ b/src/asyncrpcqueue.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2016 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "asyncrpcqueue.h" + +static std::atomic workerCounter(0); + +AsyncRPCQueue::AsyncRPCQueue() : closed(false) { +} + +/* + * Calling thread will join on all the worker threads + */ +AsyncRPCQueue::~AsyncRPCQueue() { + this->closed = true; // set this in case close() was not invoked + for (std::thread & t : this->workers) { + t.join(); + } +} + +/* + * A worker will execute this method on a new thread + */ +void AsyncRPCQueue::run(int workerId) { +// std::cout << "Launched queue worker " << workerId << std::endl; + + while (!isClosed()) { + AsyncRPCOperationId key; + std::shared_ptr operation; + { + std::unique_lock< std::mutex > guard(cs_lock); + while (operationIdQueue.empty() && !isClosed()) { + this->cs_condition.wait(guard); + } + + // Exit if the queue is closing. + if (isClosed()) + break; + + // Get operation id + key = operationIdQueue.front(); + operationIdQueue.pop(); + + // Search operation map + AsyncRPCOperationMap::const_iterator iter = operationMap.find(key); + if (iter != operationMap.end()) { + operation = iter->second; + } + } + + if (!operation) { + // cannot find operation in map, may have been removed + } else if (operation->isCancelled()) { + // skip cancelled operation + } else { + operation->main(); + } + } +// std::cout << "Terminating queue worker " << workerId << std::endl; +} + + +/* + * Add shared_ptr to operation. + * + * To retain polymorphic behaviour, i.e. main() method of derived classes is invoked, + * caller should create the shared_ptr like thi: + * + * std::shared_ptr ptr(new MyCustomAsyncRPCOperation(params)); + * + * Don't use std::make_shared(). + */ +void AsyncRPCQueue::addOperation(const std::shared_ptr &ptrOperation) { + + // Don't add if queue is closed + if (isClosed()) + return; + + AsyncRPCOperationId id = ptrOperation->getId(); + { + std::lock_guard< std::mutex > guard(cs_lock); + operationMap.emplace(id, ptrOperation); + operationIdQueue.push(id); + this->cs_condition.notify_one(); + } +} + + +std::shared_ptr AsyncRPCQueue::getOperationForId(AsyncRPCOperationId id) { + std::shared_ptr ptr; + + std::lock_guard< std::mutex > guard(cs_lock); + AsyncRPCOperationMap::const_iterator iter = operationMap.find(id); + if (iter != operationMap.end()) { + ptr = iter->second; + } + return ptr; +} + +std::shared_ptr AsyncRPCQueue::popOperationForId(AsyncRPCOperationId id) { + std::shared_ptr ptr = getOperationForId(id); + if (ptr) { + std::lock_guard< std::mutex > guard(cs_lock); + // Note: if the id still exists in the operationIdQueue, when it gets processed by a worker + // there will no operation in the map to execute, so nothing will happen. + operationMap.erase(id); + } + return ptr; +} + +bool AsyncRPCQueue::isClosed() { + return closed; +} + +void AsyncRPCQueue::close() { + this->closed = true; + cancelAllOperations(); +} + +/* + * Call cancel() on all operations + */ +void AsyncRPCQueue::cancelAllOperations() { + std::unique_lock< std::mutex > guard(cs_lock); + for (auto key : operationMap) { + key.second->cancel(); + } + this->cs_condition.notify_all(); +} + +int AsyncRPCQueue::getOperationCount() { + std::unique_lock< std::mutex > guard(cs_lock); + return operationIdQueue.size(); +} + +/* + * Spawn a worker thread + */ +void AsyncRPCQueue::addWorker() { + std::unique_lock< std::mutex > guard(cs_lock); // Todo: could just have a lock on the vector + workers.emplace_back( std::thread(&AsyncRPCQueue::run, this, ++workerCounter) ); +} + +int AsyncRPCQueue::getNumberOfWorkers() { + return workers.size(); +} diff --git a/src/asyncrpcqueue.h b/src/asyncrpcqueue.h new file mode 100644 index 00000000000..99ba45f912e --- /dev/null +++ b/src/asyncrpcqueue.h @@ -0,0 +1,61 @@ +// Copyright (c) 2014 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef ASYNCRPCQUEUE_H +#define ASYNCRPCQUEUE_H + +#include "asyncrpcoperation.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +typedef std::unordered_map > AsyncRPCOperationMap; + + +class AsyncRPCQueue { +public: + AsyncRPCQueue(); + virtual ~AsyncRPCQueue(); + + // We don't want queue to be copied or moved around + AsyncRPCQueue(AsyncRPCQueue const&) = delete; // Copy construct + AsyncRPCQueue(AsyncRPCQueue&&) = delete; // Move construct + AsyncRPCQueue& operator=(AsyncRPCQueue const&) = delete; // Copy assign + AsyncRPCQueue& operator=(AsyncRPCQueue &&) = delete; // Move assign + + void addWorker(); + int getNumberOfWorkers(); + bool isClosed(); + void close(); + void cancelAllOperations(); + int getOperationCount(); + std::shared_ptr getOperationForId(AsyncRPCOperationId); + std::shared_ptr popOperationForId(AsyncRPCOperationId); + void addOperation(const std::shared_ptr &ptrOperation); + +private: + // addWorker() will spawn a new thread on this method + void run(int workerId); + + std::mutex cs_lock; + std::condition_variable cs_condition; + bool closed; + AsyncRPCOperationMap operationMap; + std::queue operationIdQueue; + std::vector workers; +}; + +#endif + + diff --git a/src/rpcclient.cpp b/src/rpcclient.cpp index 78ae7d93057..f3d1a545431 100644 --- a/src/rpcclient.cpp +++ b/src/rpcclient.cpp @@ -98,6 +98,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "zcbenchmark", 1 }, { "zcbenchmark", 2 }, { "getblocksubsidy", 0}, + { "z_sendmany", 1}, + { "z_sendmany", 2}, { "z_importkey", 1 } }; diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 4d38a579f4c..5e9e78ce303 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -15,6 +15,9 @@ #ifdef ENABLE_WALLET #include "wallet/wallet.h" #endif +#include "asyncrpcqueue.h" + +#include #include #include @@ -49,6 +52,7 @@ static boost::thread_group* rpc_worker_group = NULL; static boost::asio::io_service::work *rpc_dummy_work = NULL; static std::vector rpc_allow_subnets; //!< List of subnets to allow RPC connections from static std::vector< boost::shared_ptr > rpc_acceptors; +static shared_ptr async_rpc_queue; static struct CRPCSignals { @@ -382,6 +386,8 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "zcrawjoinsplit", &zc_raw_joinsplit, true }, { "wallet", "zcrawreceive", &zc_raw_receive, true }, { "wallet", "zcsamplejoinsplit", &zc_sample_joinsplit, true }, + { "wallet", "z_sendmany", &z_sendmany, true }, + { "wallet", "z_getoperationstatus", &z_getoperationstatus, true }, { "wallet", "z_getnewaddress", &z_getnewaddress, true }, { "wallet", "z_listaddresses", &z_listaddresses, true }, { "wallet", "z_exportkey", &z_exportkey, true }, @@ -737,6 +743,13 @@ void StartRPCThreads() rpc_worker_group->create_thread(boost::bind(&boost::asio::io_service::run, rpc_io_service)); fRPCRunning = true; g_rpcSignals.Started(); + + // Launch at least one async rpc worker + async_rpc_queue = std::make_shared(); + async_rpc_queue->addWorker(); + async_rpc_queue->addWorker(); + async_rpc_queue->addWorker(); + } void StartDummyRPCThread() @@ -786,6 +799,10 @@ void StopRPCThreads() delete rpc_worker_group; rpc_worker_group = NULL; delete rpc_ssl_context; rpc_ssl_context = NULL; delete rpc_io_service; rpc_io_service = NULL; + + // Tells async queue to cancel all operations and shutdown. + // The async queue destructor will block and join on worker threads. + async_rpc_queue->close(); } bool IsRPCRunning() @@ -1048,3 +1065,9 @@ std::string HelpExampleRpc(string methodname, string args){ } const CRPCTable tableRPC; + +// Return async rpc queue +std::shared_ptr getAsyncRPCQueue() +{ + return async_rpc_queue; +} diff --git a/src/rpcserver.h b/src/rpcserver.h index 0e8b60b7b52..ddd4dc2beff 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -14,11 +14,13 @@ #include #include #include +#include #include "json/json_spirit_reader_template.h" #include "json/json_spirit_utils.h" #include "json/json_spirit_writer_template.h" +class AsyncRPCQueue; class CRPCCommand; namespace RPCServer @@ -55,6 +57,10 @@ void StopRPCThreads(); /** Query whether RPC is running */ bool IsRPCRunning(); +/** Get the async queue*/ +std::shared_ptr getAsyncRPCQueue(); + + /** * Set the RPC warmup status. When this is done, all RPC calls will error out * immediately with RPC_IN_WARMUP. @@ -249,6 +255,9 @@ extern json_spirit::Value z_getnewaddress(const json_spirit::Array& params, bool extern json_spirit::Value z_listaddresses(const json_spirit::Array& params, bool fHelp); // in rpcwallet.cpp extern json_spirit::Value z_exportwallet(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp extern json_spirit::Value z_importwallet(const json_spirit::Array& params, bool fHelp); // in rpcdump.cpp +extern json_spirit::Value z_sendmany(const json_spirit::Array& params, bool fHelp); // in rpcwallet.cpp +extern json_spirit::Value z_getoperationstatus(const json_spirit::Array& params, bool fHelp); // in rpcwallet.cpp + // in rest.cpp extern bool HTTPReq_REST(AcceptedConnection *conn, diff --git a/src/wallet/asyncrpcoperation_sendmany.cpp b/src/wallet/asyncrpcoperation_sendmany.cpp new file mode 100644 index 00000000000..d379a010c02 --- /dev/null +++ b/src/wallet/asyncrpcoperation_sendmany.cpp @@ -0,0 +1,77 @@ +// Copyright (c) 2016 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + + +#include "asyncrpcoperation_sendmany.h" + +#include +#include +#include + +AsyncRPCOperation_sendmany::AsyncRPCOperation_sendmany(std::string fromAddress, std::vector outputs, int minconf) : fromAddress(fromAddress), outputs(outputs), minconf(minconf) +{ +} + +AsyncRPCOperation_sendmany::AsyncRPCOperation_sendmany(const AsyncRPCOperation_sendmany& orig) { +} + +AsyncRPCOperation_sendmany::~AsyncRPCOperation_sendmany() { +} + +void AsyncRPCOperation_sendmany::main() { + if (isCancelled()) + return; + + setState(OperationStatus::EXECUTING); + startExecutionClock(); + + /** + * Dummy run of a sendmany operation + */ + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + + std::cout << std::endl << "z_sendmany: **************** DUMMY RUN *****************" << std::endl; + std::cout << "z_sendmany: source of funds: " << fromAddress << std::endl; + std::cout << "z_sendmany: minconf: " << minconf << std::endl; + + for (SendManyRecipient & t : outputs) { + std::cout << "z_sendmany: send " << std::get<1>(t) << " to " << std::get<0>(t) << std::endl; + std::string memo = std::get<2>(t); + if (memo.size()>0) { + std::cout << " : memo = " << memo << std::endl; + } + } + + std::cout << "z_sendmany: checking balances and selecting coins and notes..." << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + + std::cout << "z_sendmany: performing a joinsplit..." << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(3000)); + + std::cout << "z_sendmany: attempting to broadcast to network..." << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + + std::cout << "z_sendmany: operation complete!" << std::endl; + std::cout << "z_sendmany: ********************************************" << std::endl; + + stopExecutionClock(); + + + // dummy run will say that even number of outputs is success + bool isEven = outputs.size() % 2 == 0; + //std::cout<< "here is r: " << r << std::endl; + if (isEven) { + setState(OperationStatus::SUCCESS); + Object obj; + obj.push_back(Pair("dummy_txid", "4a1298544a1298544a1298544a1298544a129854")); + obj.push_back(Pair("dummy_fee", 0.0001)); // dummy fee + setResult(Value(obj)); + } else { + setState(OperationStatus::FAILED); + errorCode = std::rand(); + errorMessage = "Dummy run tests error handling by not liking an odd number number of outputs."; + } + +} + diff --git a/src/wallet/asyncrpcoperation_sendmany.h b/src/wallet/asyncrpcoperation_sendmany.h new file mode 100644 index 00000000000..81496c798e9 --- /dev/null +++ b/src/wallet/asyncrpcoperation_sendmany.h @@ -0,0 +1,32 @@ +// Copyright (c) 2016 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef ASYNCRPCOPERATION_SENDMANY_H +#define ASYNCRPCOPERATION_SENDMANY_H + +#include "../asyncrpcoperation.h" + +#include "amount.h" + +#include + +// A recipient is a tuple of address, amount, memo (optional if zaddr) +typedef std::tuple SendManyRecipient; + +class AsyncRPCOperation_sendmany : public AsyncRPCOperation { +public: + AsyncRPCOperation_sendmany(std::string fromAddress, std::vector outputs, int minconf); + AsyncRPCOperation_sendmany(const AsyncRPCOperation_sendmany& orig); + virtual ~AsyncRPCOperation_sendmany(); + + virtual void main(); + +private: + std::string fromAddress; + std::vector outputs; + int minconf; +}; + +#endif /* ASYNCRPCOPERATION_SENDMANY_H */ + diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e37f56b5018..3162dd56046 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -20,6 +20,10 @@ #include "zcbenchmarks.h" #include "script/interpreter.h" +#include "utiltime.h" +#include "asyncrpcoperation.h" +#include "wallet/asyncrpcoperation_sendmany.h" + #include "sodium.h" #include @@ -28,6 +32,7 @@ #include "json/json_spirit_utils.h" #include "json/json_spirit_value.h" +#include "asyncrpcqueue.h" using namespace std; using namespace json_spirit; @@ -2827,3 +2832,189 @@ Value z_listaddresses(const Array& params, bool fHelp) return ret; } + +Value z_getoperationstatus(const Array& params, bool fHelp) +{ + if (!EnsureWalletIsAvailable(fHelp)) + return Value::null; + + if (fHelp || params.size() != 1) + throw runtime_error( + "z_getoperationstatus \"operationid\"\n" + "\nGet operation status and any associated result or error data." + + HelpRequiringPassphrase() + "\n" + "\nArguments:\n" + "1. \"operationid\" (string, required) The operation id returned by an async operation call.\n" + "\nResult:\n" + "\" object\" (string) FIXME: ASYNC operation object \n" + " with some key value pairs.\n" + ); + + LOCK2(cs_main, pwalletMain->cs_wallet); + + AsyncRPCOperationId id = params[0].get_str(); + + std::shared_ptr q = getAsyncRPCQueue(); + std::shared_ptr operation = q->getOperationForId(id); + if (!operation) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "No operation exists for that id."); + } + + Value status = operation->getStatus(); + + // Remove operation from memory when it has finished and the caller has retrieved the result and reason for finishing. + if (operation->isSuccess() || operation->isFailed() || operation->isCancelled()) { + q->popOperationForId(id); + } + + return status; +} + + +Value z_sendmany(const Array& params, bool fHelp) +{ + if (!EnsureWalletIsAvailable(fHelp)) + return Value::null; + + if (fHelp || params.size() < 2 || params.size() > 3) + throw runtime_error( + "z_sendmany \"fromaddress\" [{\"address\":... ,\"amount\":...},...] ( minconf )\n" + "\n*** THIS COMMAND HAS NOT BEEN IMPLEMENTED YET ***" + "\nSend multiple times. Amounts are double-precision floating point numbers." + + HelpRequiringPassphrase() + "\n" + "\nArguments:\n" + "1. \"fromaddress\" (string, required) The taddr or zaddr to send the funds from.\n" + "2. \"amounts\" (array, required) An array of json objects representing the amounts to send.\n" + " [{\n" + " \"address\":address (string, required) The address is a taddr or zaddr\n" + " \"amount\":amount (numeric, required) The numeric amount in ZEC is the value\n" + " \"memo\":memo (string, optional) If the address is a zaddr, raw data represented in hexadecimal string format\n" + " }, ... ]\n" + "3. minconf (numeric, optional, default=1) Only use funds confirmed at least this many times.\n" + "\nResult:\n" + "\"operationid\" (string) An operationid to pass to z_getoperationstatus to get the result of the operation.\n" + ); + + LOCK2(cs_main, pwalletMain->cs_wallet); + + + // Check that the from address is valid. + auto fromaddress = params[0].get_str(); + bool fromTaddr = false; + CBitcoinAddress taddr(fromaddress); + fromTaddr = taddr.IsValid(); + libzcash::PaymentAddress zaddr; + if (!fromTaddr) { + CZCPaymentAddress address(fromaddress); + try { + zaddr = address.Get(); + } catch (std::runtime_error) { + // invalid + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid from address, should be a taddr or zaddr."); + } + } + + // Check that we have the spending key? + if (!fromTaddr) { + if (!pwalletMain->HaveSpendingKey(zaddr)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "From address does not belong to this node, zaddr spending key not found."); + } + } + + + Array outputs = params[1].get_array(); + + if (outputs.size()==0) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, amounts array is empty."); + + // Keep track of addresses to spot duplicates + set setAddress; + + // Recipients + std::vector recipients; + + BOOST_FOREACH(Value& output, outputs) + { + if (output.type() != obj_type) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected object"); + const Object& o = output.get_obj(); + + RPCTypeCheck(o, boost::assign::map_list_of("address", str_type)("amount", real_type)); + + // sanity check, report error if unknown key-value pairs +// for (auto& p : o) { + for (const Pair& p : o) { + std::string s = p.name_; + if (s != "address" && s != "amount" && s!="memo") + throw JSONRPCError(RPC_INVALID_PARAMETER, string("Invalid parameter, unknown key: ")+s); + } + + string address = find_value(o, "address").get_str(); + bool isZaddr = false; + CBitcoinAddress taddr(address); + if (!taddr.IsValid()) { + try { + CZCPaymentAddress zaddr(address); + zaddr.Get(); + isZaddr = true; + } catch (std::runtime_error) { + throw JSONRPCError(RPC_INVALID_PARAMETER, string("Invalid parameter, unknown address format: ")+address ); + } + } + + if (setAddress.count(address)) + throw JSONRPCError(RPC_INVALID_PARAMETER, string("Invalid parameter, duplicated address: ")+address); + setAddress.insert(address); + + Value memoValue = find_value(o, "memo"); + string memo; + if (!memoValue.is_null()) { + memo = memoValue.get_str(); + if (!isZaddr) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Memo can not be used with a taddr. It can only be used with a zaddr."); + } else if (!IsHex(memo)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected memo data in hexadecimal format."); + } + } + + //int nOutput = find_value(o, "amount").get_real(); // int(); + Value av = find_value(o, "amount"); + CAmount nAmount = AmountFromValue( av ); + if (nAmount < 0) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, amount must be positive"); + + recipients.push_back( SendManyRecipient(address, nAmount, memo) ); + } + + + + // Minimum confirmations + int nMinDepth = 1; + if (params.size() > 2) + nMinDepth = params[2].get_int(); + + +// std::vector +// GetPaymentAddresses(addresses); +// for (auto addr : addresses ) { +// ret.push_back(CZCPaymentAddress(addr).ToString()); +// } + + + std::shared_ptr q = getAsyncRPCQueue(); + std::shared_ptr operation( new AsyncRPCOperation_sendmany(fromaddress, recipients, nMinDepth) ); + // operation-> + //std::shared_ptr operation = make_shared(AsyncRPCOperation_sendmany()); + q->addOperation(operation); + AsyncRPCOperationId operationId = operation->getId(); + return operationId; +// return Value::null; + +// Array ret; +// std::set addresses; +// pwalletMain->GetPaymentAddresses(addresses); +// for (auto addr : addresses ) { +// ret.push_back(CZCPaymentAddress(addr).ToString()); +// } +// return ret; +} From e2898aa82d653233388f99dd71fc612c55cfc591 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 7 Aug 2016 23:54:50 -0700 Subject: [PATCH 12/22] Fixes #1193 so that during verification benchmarking it does not unncessarily create thousands of CTransaction objects. --- src/zcbenchmarks.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/zcbenchmarks.cpp b/src/zcbenchmarks.cpp index c7460fb355f..9a16f0be65a 100644 --- a/src/zcbenchmarks.cpp +++ b/src/zcbenchmarks.cpp @@ -193,6 +193,9 @@ double benchmark_large_tx() assert(ss.size() > MAX_BLOCK_SIZE - error); } + // Spending tx has all its inputs signed and does not need to be mutated anymore + CTransaction final_spending_tx(spending_tx); + // Benchmark signature verification costs: timer_start(); for (size_t i = 0; i < NUM_INPUTS; i++) { @@ -200,7 +203,7 @@ double benchmark_large_tx() assert(VerifyScript(spending_tx.vin[i].scriptSig, prevPubKey, STANDARD_SCRIPT_VERIFY_FLAGS, - MutableTransactionSignatureChecker(&spending_tx, i), + TransactionSignatureChecker(&final_spending_tx, i), &serror)); } return timer_stop(); From 2c27fb20815fd30f1b4d919e49d94fcec4fb5363 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 8 Aug 2016 00:51:27 -0700 Subject: [PATCH 13/22] Update variable. --- src/zcbenchmarks.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zcbenchmarks.cpp b/src/zcbenchmarks.cpp index 9a16f0be65a..94f27ae6bda 100644 --- a/src/zcbenchmarks.cpp +++ b/src/zcbenchmarks.cpp @@ -200,7 +200,7 @@ double benchmark_large_tx() timer_start(); for (size_t i = 0; i < NUM_INPUTS; i++) { ScriptError serror = SCRIPT_ERR_OK; - assert(VerifyScript(spending_tx.vin[i].scriptSig, + assert(VerifyScript(final_spending_tx.vin[i].scriptSig, prevPubKey, STANDARD_SCRIPT_VERIFY_FLAGS, TransactionSignatureChecker(&final_spending_tx, i), From c90e1f080503b2924c545c3bbdce7bd47398d6a1 Mon Sep 17 00:00:00 2001 From: "Robert C. Seacord" Date: Wed, 10 Aug 2016 15:40:32 -0400 Subject: [PATCH 14/22] Update equihash.cpp fix https://github.com/zcash/zcash/issues/1214 --- src/crypto/equihash.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crypto/equihash.cpp b/src/crypto/equihash.cpp index b387a75b001..9197037cec3 100644 --- a/src/crypto/equihash.cpp +++ b/src/crypto/equihash.cpp @@ -200,7 +200,7 @@ TruncatedStepRow& TruncatedStepRow::operator=(const TruncatedStepR template std::shared_ptr TruncatedStepRow::GetTruncatedIndices(size_t len, size_t lenIndices) const { - std::shared_ptr p (new eh_trunc[lenIndices]); + std::shared_ptr p (new eh_trunc[lenIndices], std::default_delete()); std::copy(hash+len, hash+len+lenIndices, p.get()); return p; } From 67016997865b9d35ce28f1ed9263e209a68fc157 Mon Sep 17 00:00:00 2001 From: "Robert C. Seacord" Date: Wed, 10 Aug 2016 22:02:00 -0400 Subject: [PATCH 15/22] Update test_equihash.cpp --- src/gtest/test_equihash.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gtest/test_equihash.cpp b/src/gtest/test_equihash.cpp index 81ddf190861..15d7ac52b32 100644 --- a/src/gtest/test_equihash.cpp +++ b/src/gtest/test_equihash.cpp @@ -4,9 +4,9 @@ #include "crypto/equihash.h" TEST(equihash_tests, is_probably_duplicate) { - std::shared_ptr p1 (new eh_trunc[4] {0, 1, 2, 3}); - std::shared_ptr p2 (new eh_trunc[4] {0, 1, 1, 3}); - std::shared_ptr p3 (new eh_trunc[4] {3, 1, 1, 3}); + std::shared_ptr p1 (new eh_trunc[4] {0, 1, 2, 3}, std::default_delete()); + std::shared_ptr p2 (new eh_trunc[4] {0, 1, 1, 3}, std::default_delete()); + std::shared_ptr p3 (new eh_trunc[4] {3, 1, 1, 3}, std::default_delete()); ASSERT_FALSE(IsProbablyDuplicate<4>(p1, 4)); ASSERT_FALSE(IsProbablyDuplicate<4>(p2, 4)); From 54c9d68a7ed47e68cc93775cf35f3654633706cc Mon Sep 17 00:00:00 2001 From: Taylor Hornby Date: Thu, 11 Aug 2016 15:15:50 -0600 Subject: [PATCH 16/22] Disable hardening when building for coverage reports. --- zcutil/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zcutil/build.sh b/zcutil/build.sh index 6f79157f7ea..983ee8c938c 100755 --- a/zcutil/build.sh +++ b/zcutil/build.sh @@ -24,9 +24,11 @@ cd "$(dirname "$(readlink -f "$0")")/.." # If --enable-lcov is the first argument, enable lcov coverage support: LCOV_ARG='' +HARDENING_ARG='--enable-hardening' if [ "x${1:-}" = 'x--enable-lcov' ] then LCOV_ARG='--enable-lcov' + HARDENING_ARG='--disable-hardening' shift fi @@ -35,5 +37,5 @@ PREFIX="$(pwd)/depends/x86_64-unknown-linux-gnu/" make "$@" -C ./depends/ V=1 NO_QT=1 ./autogen.sh -./configure --prefix="${PREFIX}" --with-gui=no --enable-hardening "$LCOV_ARG" CXXFLAGS='-Wno-deprecated-declarations -Wno-placement-new -Wno-terminate -Werror -O1 -g' +./configure --prefix="${PREFIX}" --with-gui=no "$HARDENING_ARG" "$LCOV_ARG" CXXFLAGS='-Wno-deprecated-declarations -Wno-placement-new -Wno-terminate -Werror -O1 -g' make "$@" V=1 From aa06d4a55ca2252c081d7b0d429e1a66cab88c70 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 18 Aug 2016 12:22:30 +1200 Subject: [PATCH 17/22] Rework test to check for failure to return a spending key --- src/gtest/test_keystore.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/gtest/test_keystore.cpp b/src/gtest/test_keystore.cpp index 8baf02cab58..3d095ee56f4 100644 --- a/src/gtest/test_keystore.cpp +++ b/src/gtest/test_keystore.cpp @@ -5,19 +5,22 @@ TEST(keystore_tests, store_and_retrieve_spending_key) { CBasicKeyStore keyStore; + libzcash::SpendingKey skOut; std::set addrs; keyStore.GetPaymentAddresses(addrs); ASSERT_EQ(0, addrs.size()); auto sk = libzcash::SpendingKey::random(); - keyStore.AddSpendingKey(sk); - auto addr = sk.address(); - ASSERT_TRUE(keyStore.HaveSpendingKey(addr)); - libzcash::SpendingKey skOut; - keyStore.GetSpendingKey(addr, skOut); + // Sanity-check: we can't get a key we haven't added + ASSERT_FALSE(keyStore.HaveSpendingKey(addr)); + ASSERT_FALSE(keyStore.GetSpendingKey(addr, skOut)); + + keyStore.AddSpendingKey(sk); + ASSERT_TRUE(keyStore.HaveSpendingKey(addr)); + ASSERT_TRUE(keyStore.GetSpendingKey(addr, skOut)); ASSERT_EQ(sk, skOut); keyStore.GetPaymentAddresses(addrs); From b51672ef508df574acedd5240369fd9e57422363 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 18 Aug 2016 12:25:01 +1200 Subject: [PATCH 18/22] ASSERT -> EXPECT in test to get more info per test run about future regressions --- src/gtest/test_keystore.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/gtest/test_keystore.cpp b/src/gtest/test_keystore.cpp index 3d095ee56f4..8587ba49b49 100644 --- a/src/gtest/test_keystore.cpp +++ b/src/gtest/test_keystore.cpp @@ -9,21 +9,21 @@ TEST(keystore_tests, store_and_retrieve_spending_key) { std::set addrs; keyStore.GetPaymentAddresses(addrs); - ASSERT_EQ(0, addrs.size()); + EXPECT_EQ(0, addrs.size()); auto sk = libzcash::SpendingKey::random(); auto addr = sk.address(); // Sanity-check: we can't get a key we haven't added - ASSERT_FALSE(keyStore.HaveSpendingKey(addr)); - ASSERT_FALSE(keyStore.GetSpendingKey(addr, skOut)); + EXPECT_FALSE(keyStore.HaveSpendingKey(addr)); + EXPECT_FALSE(keyStore.GetSpendingKey(addr, skOut)); keyStore.AddSpendingKey(sk); - ASSERT_TRUE(keyStore.HaveSpendingKey(addr)); - ASSERT_TRUE(keyStore.GetSpendingKey(addr, skOut)); - ASSERT_EQ(sk, skOut); + EXPECT_TRUE(keyStore.HaveSpendingKey(addr)); + EXPECT_TRUE(keyStore.GetSpendingKey(addr, skOut)); + EXPECT_EQ(sk, skOut); keyStore.GetPaymentAddresses(addrs); - ASSERT_EQ(1, addrs.size()); - ASSERT_EQ(1, addrs.count(addr)); + EXPECT_EQ(1, addrs.size()); + EXPECT_EQ(1, addrs.count(addr)); } From 761d2a864dbf537e04b81473fa2a1d53e4b7a801 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 12 Aug 2016 01:27:20 +1200 Subject: [PATCH 19/22] Add wallet method for finding spendable Notes in a CTransaction --- src/Makefile.gtest.include | 1 + src/gtest/test_wallet.cpp | 30 ++++++++++++++++++++++++++++++ src/wallet/wallet.cpp | 33 +++++++++++++++++++++++++++++++++ src/wallet/wallet.h | 8 ++++++++ src/zcash/Address.hpp | 1 + 5 files changed, 73 insertions(+) create mode 100644 src/gtest/test_wallet.cpp diff --git a/src/Makefile.gtest.include b/src/Makefile.gtest.include index 2f5fc596dbf..6ea153a3609 100644 --- a/src/Makefile.gtest.include +++ b/src/Makefile.gtest.include @@ -13,6 +13,7 @@ zcash_gtest_SOURCES = \ gtest/test_merkletree.cpp \ gtest/test_circuit.cpp \ gtest/test_txid.cpp \ + gtest/test_wallet.cpp \ gtest/test_wallet_zkeys.cpp \ gtest/test_libzcash_utils.cpp diff --git a/src/gtest/test_wallet.cpp b/src/gtest/test_wallet.cpp new file mode 100644 index 00000000000..b94ecb053ab --- /dev/null +++ b/src/gtest/test_wallet.cpp @@ -0,0 +1,30 @@ +#include + +#include "base58.h" +#include "chainparams.h" +#include "wallet/wallet.h" + +CZCSpendingKey testSpendingKey("TKWTVvdbYaXSa1Red2X7PgpoomQnDZMPECKDqn5QhTjDvCabvbr8"); +// Raw tx taking transparent input and spending it to two addresses: +// 0: TKWREumuAmKX3xJJ5MadYWqwtD76msJgYyEBp6vLLX5W1HzgcEQX +// 1: TKWTVvdbYaXSa1Red2X7PgpoomQnDZMPECKDqn5QhTjDvCabvbr8 +const auto testRawJS = ParseHex("02000000012dc18a2c0993c343d892deb9e81ffb5ce016079c4ecdca4f80299579242c99030000000000ffffffff000000000001a0f95600000000000000000000000000a54715500c63cd37d6a1a6e8fb9ccac887aa8c8b3fe5672a08335719c942aa2b2e5163cb8da7c25e887debbc9d357dc6a0f5484547f2fcc78b1f44cb54e19d274e6371b8325888bc01b1b951098f407346b3f9ac6393b6e6b50b6d419a41f05a8534a3d91df4c708fb0830cc240e93ee42012aaf6c5f1d600a2104630c618c9624f9c80eef3ece7b7a59b10e905903eac9ed66bb1619a53bff4afe72153d51597be499ecff8a8730f4c0d20e4694635069e6ffb1212f5009f9243a4bc065ea6da6752bbbe0e0b2a148c71cdc02aaa55618ea7ad7e5105be30b8c2c7d24b6ed436e55885d91c36b497d80b21397da740fb3f8dbc09c3387a1085331d9e860ad741fd5300131e74c42df6ebef1f5acce28c89b8c3b48e7f802887f4a3b53f6148038be98c56a72dd39995c3932209ea60dd198ccd2308bff4a77a2851fe8ff68df7ebe64f6512e14a8ed874229bd4c66a746f7b3be62a9d97574bc53773508216d1281e5234f46792c8ee200f38542264b480352c4b24f08ea60b2fa11dc21af1eff5058fc53a9d23bac98548597bbaf15797b9d8f5c413919988d6bc8d0c709d6f98d12a863b0c2a8dcd2ce2f4aef285b0561e37de6b8c338ffb6a8e505eb570a3418c1c5050ef6aaf2c68f77b07b97370f7a22783a93eeebcdb37278c40e8367bf20dcd13b1ad39bbc15dca1725147395d0a393c7efe9fef15961044761febc359d575cd1d80abdf554a0fee0e61483dc1a4a8de1b378dbd32bc386b13fd738688d5989f1e4dcdad701a4933f40aeba9d04299ea8f1b73c8b6e4486c0a3cc669d31acb06d49cf42e8fa2aaca0f67d9aa3ec650fc6689de7a48f4b8eeaf70e4539b9b11f68c448a9dd2423e2b37fec85b1cdb138b7438148a0fc8e4301a32ccceaae486d26675e4605d93b05ed28880ce4e4f36dfab80909ebe0e7c779ce6adda8525d164612c6d0cba43c27ca0e37e6fa4dbdb2c465e561e9e5c696afd4a0ebbdfe978503667f7629925e99c706380980f8730ce3c67976945922ef4769e58a44fb3a7f1459c6321867566180d1e335b3d4f2d3d966bb1090c3cdd4d3a7a4d9ca2ff58d9137c312aec8acbb3e029b691a9f60f30b8304e1a2f1f9641adf6997ebf85b7e3bdfd73e1e3c24568b05e6e6a66d4df1e287f3e6bc124cc04dc011d09d145c2a823cf6ff6ac942df939250f8d9a088f2c301a98b3e878313e30631a6cd07999e7852a877e4930db2a5cd9708172f023e322ff7aa5192fe336a5b9881a67e0d63102f99b33288056db2d8b03a358a26b2e18fb906f60cbdc476bda2c837d04f489d7ac5862058eddf97b10ef39258ed8af0167609b94097b9eeb313bbe468f2e627da740d4741c6512dfbdc84e5a31a795013082d94faff20e4483d9334e13909c5d54e3bbb1d26447e0b3eccc5499b98119289c8613ffca3f4053c69778b3cfbcc3d9ffc53741081d328bedf6dc83ee56461130f31fbf14acdfb43d796ce4ea86a015ab3647760f292958ce4c9ae9c6fc0f3120fece3bae378b51e7813484a893db74797e9327e3eeab1e7b7856e9cc6124460330299fd4484dd8b6b73073679a859461ec2f5ded61e9de9ed367c6c4e95d68390e78690851b8b39e19ed73798ee93a78cddab362059726228d9fb418f85c149c09306b73ef9f9b1cc8c1c9a5be9a96a21137a1bdc86ac8972666495f28ccc0adae17c874b79c26a9144573213bb44b8412ef1a0202ee653df65777f7522f30da660630786efb24a28d8e4a79dc0e4378276ecec16b0462aff7295c025d84d4c51f8b2c3ff5377b9657cf0160112d1586aae6c1ce8f3c8fcd599342b73dd282a43a1024fa94ecb630df5f8017d19f3c8edfecdd2ab5886db8403ca4a1df6fd2a5194810c2ea00cfed6fdeb080c8c7b2dff67c40755cb7959be8d1d7120ff519145c5da33b516c34ef216d657ebba30ff9a9e06f2eab06724ee26e06f5f0813eb105620e"); + +TEST(wallet_tests, find_note_in_tx) { + SelectParams(CBaseChainParams::TESTNET); + CWallet wallet; + + libzcash::SpendingKey sk = testSpendingKey.Get(); + wallet.AddSpendingKey(sk); + + CDataStream ss(testRawJS, SER_NETWORK, PROTOCOL_VERSION); + CTransaction tx; + ss >> tx; + + auto noteMap = wallet.FindMyNotes(tx); + ASSERT_EQ(1, noteMap.size()); + + auto noteIndex = std::make_pair(0, 1); + ASSERT_EQ(1, noteMap.count(noteIndex)); + ASSERT_EQ(sk.address(), noteMap[noteIndex]); +} diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 0f6eeff49fd..087a40f6f7f 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -10,6 +10,7 @@ #include "coincontrol.h" #include "consensus/consensus.h" #include "consensus/validation.h" +#include "init.h" #include "main.h" #include "net.h" #include "script/script.h" @@ -17,6 +18,7 @@ #include "timedata.h" #include "util.h" #include "utilmoneystr.h" +#include "zcash/Note.hpp" #include @@ -843,6 +845,37 @@ void CWallet::EraseFromWallet(const uint256 &hash) } +mapNoteAddrs_t CWallet::FindMyNotes(const CTransaction& tx) const +{ + mapNoteAddrs_t noteAddrs; + std::set addresses; + GetPaymentAddresses(addresses); + libzcash::SpendingKey key; + for (size_t i = 0; i < tx.vjoinsplit.size(); i++) { + auto hSig = tx.vjoinsplit[i].h_sig(*pzcashParams, tx.joinSplitPubKey); + for (size_t j = 0; j < tx.vjoinsplit[i].ciphertexts.size(); j++) { + BOOST_FOREACH(const libzcash::PaymentAddress& address, addresses) + { + GetSpendingKey(address, key); + ZCNoteDecryption decryptor(key.viewing_key()); + try { + libzcash::NotePlaintext::decrypt( + decryptor, + tx.vjoinsplit[i].ciphertexts[j], + tx.vjoinsplit[i].ephemeralKey, + hSig, + (unsigned char) j); + noteAddrs.insert(std::make_pair(pNoteIndex_t(i, j), address)); + break; + } catch (const std::exception &) { + // Couldn't decrypt with this spending key + } + } + } + } + return noteAddrs; +} + isminetype CWallet::IsMine(const CTxIn &txin) const { { diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 0651c640444..723d8f828db 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -118,7 +118,13 @@ struct CRecipient bool fSubtractFeeFromAmount; }; +// Index for a particular Note in a transaction +// first: Index into CTransaction.vjoinsplit +// second: Index into JSDescription.ciphertexts +typedef std::pair pNoteIndex_t; + typedef std::map mapValue_t; +typedef std::map mapNoteAddrs_t; static void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) @@ -667,6 +673,8 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface std::set GetAccountAddresses(std::string strAccount) const; + mapNoteAddrs_t FindMyNotes(const CTransaction& tx) const; + isminetype IsMine(const CTxIn& txin) const; CAmount GetDebit(const CTxIn& txin, const isminefilter& filter) const; isminetype IsMine(const CTxOut& txout) const; diff --git a/src/zcash/Address.hpp b/src/zcash/Address.hpp index 36b9402a38a..d967aef271f 100644 --- a/src/zcash/Address.hpp +++ b/src/zcash/Address.hpp @@ -23,6 +23,7 @@ class PaymentAddress { READWRITE(pk_enc); } + friend inline bool operator==(const PaymentAddress& a, const PaymentAddress& b) { return a.a_pk == b.a_pk && a.pk_enc == b.pk_enc; } friend inline bool operator<(const PaymentAddress& a, const PaymentAddress& b) { return a.a_pk < b.a_pk; } }; From a423578c9004cea35931800e9ced7c4d8e9eb46c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 12 Aug 2016 02:11:46 +1200 Subject: [PATCH 20/22] Store mapping between Notes and PaymentAddresses in CWalletTx --- src/gtest/test_wallet.cpp | 29 +++++++++++++++++++++++++++++ src/wallet/wallet.cpp | 17 ++++++++++++++++- src/wallet/wallet.h | 5 +++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/gtest/test_wallet.cpp b/src/gtest/test_wallet.cpp index b94ecb053ab..959b5c311e9 100644 --- a/src/gtest/test_wallet.cpp +++ b/src/gtest/test_wallet.cpp @@ -10,6 +10,35 @@ CZCSpendingKey testSpendingKey("TKWTVvdbYaXSa1Red2X7PgpoomQnDZMPECKDqn5QhTjDvCab // 1: TKWTVvdbYaXSa1Red2X7PgpoomQnDZMPECKDqn5QhTjDvCabvbr8 const auto testRawJS = ParseHex("02000000012dc18a2c0993c343d892deb9e81ffb5ce016079c4ecdca4f80299579242c99030000000000ffffffff000000000001a0f95600000000000000000000000000a54715500c63cd37d6a1a6e8fb9ccac887aa8c8b3fe5672a08335719c942aa2b2e5163cb8da7c25e887debbc9d357dc6a0f5484547f2fcc78b1f44cb54e19d274e6371b8325888bc01b1b951098f407346b3f9ac6393b6e6b50b6d419a41f05a8534a3d91df4c708fb0830cc240e93ee42012aaf6c5f1d600a2104630c618c9624f9c80eef3ece7b7a59b10e905903eac9ed66bb1619a53bff4afe72153d51597be499ecff8a8730f4c0d20e4694635069e6ffb1212f5009f9243a4bc065ea6da6752bbbe0e0b2a148c71cdc02aaa55618ea7ad7e5105be30b8c2c7d24b6ed436e55885d91c36b497d80b21397da740fb3f8dbc09c3387a1085331d9e860ad741fd5300131e74c42df6ebef1f5acce28c89b8c3b48e7f802887f4a3b53f6148038be98c56a72dd39995c3932209ea60dd198ccd2308bff4a77a2851fe8ff68df7ebe64f6512e14a8ed874229bd4c66a746f7b3be62a9d97574bc53773508216d1281e5234f46792c8ee200f38542264b480352c4b24f08ea60b2fa11dc21af1eff5058fc53a9d23bac98548597bbaf15797b9d8f5c413919988d6bc8d0c709d6f98d12a863b0c2a8dcd2ce2f4aef285b0561e37de6b8c338ffb6a8e505eb570a3418c1c5050ef6aaf2c68f77b07b97370f7a22783a93eeebcdb37278c40e8367bf20dcd13b1ad39bbc15dca1725147395d0a393c7efe9fef15961044761febc359d575cd1d80abdf554a0fee0e61483dc1a4a8de1b378dbd32bc386b13fd738688d5989f1e4dcdad701a4933f40aeba9d04299ea8f1b73c8b6e4486c0a3cc669d31acb06d49cf42e8fa2aaca0f67d9aa3ec650fc6689de7a48f4b8eeaf70e4539b9b11f68c448a9dd2423e2b37fec85b1cdb138b7438148a0fc8e4301a32ccceaae486d26675e4605d93b05ed28880ce4e4f36dfab80909ebe0e7c779ce6adda8525d164612c6d0cba43c27ca0e37e6fa4dbdb2c465e561e9e5c696afd4a0ebbdfe978503667f7629925e99c706380980f8730ce3c67976945922ef4769e58a44fb3a7f1459c6321867566180d1e335b3d4f2d3d966bb1090c3cdd4d3a7a4d9ca2ff58d9137c312aec8acbb3e029b691a9f60f30b8304e1a2f1f9641adf6997ebf85b7e3bdfd73e1e3c24568b05e6e6a66d4df1e287f3e6bc124cc04dc011d09d145c2a823cf6ff6ac942df939250f8d9a088f2c301a98b3e878313e30631a6cd07999e7852a877e4930db2a5cd9708172f023e322ff7aa5192fe336a5b9881a67e0d63102f99b33288056db2d8b03a358a26b2e18fb906f60cbdc476bda2c837d04f489d7ac5862058eddf97b10ef39258ed8af0167609b94097b9eeb313bbe468f2e627da740d4741c6512dfbdc84e5a31a795013082d94faff20e4483d9334e13909c5d54e3bbb1d26447e0b3eccc5499b98119289c8613ffca3f4053c69778b3cfbcc3d9ffc53741081d328bedf6dc83ee56461130f31fbf14acdfb43d796ce4ea86a015ab3647760f292958ce4c9ae9c6fc0f3120fece3bae378b51e7813484a893db74797e9327e3eeab1e7b7856e9cc6124460330299fd4484dd8b6b73073679a859461ec2f5ded61e9de9ed367c6c4e95d68390e78690851b8b39e19ed73798ee93a78cddab362059726228d9fb418f85c149c09306b73ef9f9b1cc8c1c9a5be9a96a21137a1bdc86ac8972666495f28ccc0adae17c874b79c26a9144573213bb44b8412ef1a0202ee653df65777f7522f30da660630786efb24a28d8e4a79dc0e4378276ecec16b0462aff7295c025d84d4c51f8b2c3ff5377b9657cf0160112d1586aae6c1ce8f3c8fcd599342b73dd282a43a1024fa94ecb630df5f8017d19f3c8edfecdd2ab5886db8403ca4a1df6fd2a5194810c2ea00cfed6fdeb080c8c7b2dff67c40755cb7959be8d1d7120ff519145c5da33b516c34ef216d657ebba30ff9a9e06f2eab06724ee26e06f5f0813eb105620e"); +TEST(wallet_tests, set_note_addrs_in_cwallettx) { + SelectParams(CBaseChainParams::TESTNET); + + CDataStream ss(testRawJS, SER_NETWORK, PROTOCOL_VERSION); + CTransaction tx; + ss >> tx; + CWalletTx wtx {NULL, tx}; + ASSERT_EQ(0, wtx.mapNoteAddrs.size()); + + mapNoteAddrs_t noteAddrs; + libzcash::SpendingKey sk = testSpendingKey.Get(); + noteAddrs[std::make_pair(0, 1)] = sk.address(); + + wtx.SetNoteAddresses(noteAddrs); + ASSERT_EQ(noteAddrs, wtx.mapNoteAddrs); +} + +TEST(wallet_tests, set_invalid_note_addrs_in_cwallettx) { + CWalletTx wtx; + ASSERT_EQ(0, wtx.mapNoteAddrs.size()); + + mapNoteAddrs_t noteAddrs; + auto sk = libzcash::SpendingKey::random(); + noteAddrs[std::make_pair(0, 1)] = sk.address(); + + wtx.SetNoteAddresses(noteAddrs); + ASSERT_EQ(0, wtx.mapNoteAddrs.size()); +} + TEST(wallet_tests, find_note_in_tx) { SelectParams(CBaseChainParams::TESTNET); CWallet wallet; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 087a40f6f7f..9dde25850d0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -798,7 +798,8 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlock* pbl AssertLockHeld(cs_wallet); bool fExisted = mapWallet.count(tx.GetTxid()) != 0; if (fExisted && !fUpdate) return false; - if (fExisted || IsMine(tx) || IsFromMe(tx)) + auto noteAddrs = FindMyNotes(tx); + if (fExisted || IsMine(tx) || IsFromMe(tx) || noteAddrs.size() > 0) { CWalletTx wtx(this,tx); @@ -806,6 +807,9 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlock* pbl if (pblock) wtx.SetMerkleBranch(*pblock); + if (noteAddrs.size() > 0) + wtx.SetNoteAddresses(noteAddrs); + // Do not flush the wallet here for performance reasons // this is safe, as in case of a crash, we rescan the necessary blocks on startup through our SetBestChain-mechanism CWalletDB walletdb(strWalletFile, "r+", false); @@ -997,6 +1001,17 @@ CAmount CWallet::GetChange(const CTransaction& tx) const return nChange; } +void CWalletTx::SetNoteAddresses(mapNoteAddrs_t ¬eAddrs) +{ + mapNoteAddrs.clear(); + for (const std::pair n : noteAddrs) { + if (n.first.first < vjoinsplit.size() && + n.first.second < vjoinsplit[n.first.first].ciphertexts.size()) { + mapNoteAddrs[n.first] = n.second; + } // TODO ignore invalid note indices, or error? + } +} + int64_t CWalletTx::GetTxTime() const { int64_t n = nTimeSmart; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 723d8f828db..a827be4775b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -222,6 +222,7 @@ class CWalletTx : public CMerkleTx public: mapValue_t mapValue; + mapNoteAddrs_t mapNoteAddrs; std::vector > vOrderForm; unsigned int fTimeReceivedIsTxTime; unsigned int nTimeReceived; //! time received by this node @@ -274,6 +275,7 @@ class CWalletTx : public CMerkleTx { pwallet = pwalletIn; mapValue.clear(); + mapNoteAddrs.clear(); vOrderForm.clear(); fTimeReceivedIsTxTime = false; nTimeReceived = 0; @@ -323,6 +325,7 @@ class CWalletTx : public CMerkleTx std::vector vUnused; //! Used to be vtxPrev READWRITE(vUnused); READWRITE(mapValue); + READWRITE(mapNoteAddrs); READWRITE(vOrderForm); READWRITE(fTimeReceivedIsTxTime); READWRITE(nTimeReceived); @@ -364,6 +367,8 @@ class CWalletTx : public CMerkleTx MarkDirty(); } + void SetNoteAddresses(mapNoteAddrs_t ¬eAddrs); + //! filter decides which addresses will count towards the debit CAmount GetDebit(const isminefilter& filter) const; CAmount GetCredit(const isminefilter& filter) const; From f91ea7a34b194d19e7a10d38b57a4650d5cdb7d7 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 12 Aug 2016 12:46:37 +1200 Subject: [PATCH 21/22] Cache ZCNoteDecryption objects for efficiency --- src/keystore.cpp | 4 +++- src/keystore.h | 16 ++++++++++++++++ src/wallet/wallet.cpp | 27 ++++++++++++++------------- src/zcash/NoteEncryption.hpp | 2 ++ 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/keystore.cpp b/src/keystore.cpp index 7240cd74733..251e47e2602 100644 --- a/src/keystore.cpp +++ b/src/keystore.cpp @@ -87,6 +87,8 @@ bool CBasicKeyStore::HaveWatchOnly() const bool CBasicKeyStore::AddSpendingKey(const libzcash::SpendingKey &sk) { LOCK(cs_SpendingKeyStore); - mapSpendingKeys[sk.address()] = sk; + auto address = sk.address(); + mapSpendingKeys[address] = sk; + mapNoteDecryptors[address] = ZCNoteDecryption(sk.viewing_key()); return true; } diff --git a/src/keystore.h b/src/keystore.h index 987f32070f7..ff075cead1e 100644 --- a/src/keystore.h +++ b/src/keystore.h @@ -12,6 +12,7 @@ #include "script/standard.h" #include "sync.h" #include "zcash/Address.hpp" +#include "zcash/NoteEncryption.hpp" #include #include @@ -60,6 +61,7 @@ typedef std::map KeyMap; typedef std::map ScriptMap; typedef std::set WatchOnlySet; typedef std::map SpendingKeyMap; +typedef std::map NoteDecryptorMap; /** Basic key store, that keeps keys in an address->secret map */ class CBasicKeyStore : public CKeyStore @@ -69,6 +71,7 @@ class CBasicKeyStore : public CKeyStore ScriptMap mapScripts; WatchOnlySet setWatchOnly; SpendingKeyMap mapSpendingKeys; + NoteDecryptorMap mapNoteDecryptors; public: bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey); @@ -139,6 +142,19 @@ class CBasicKeyStore : public CKeyStore } return false; } + bool GetNoteDecryptor(const libzcash::PaymentAddress &address, ZCNoteDecryption &decOut) const + { + { + LOCK(cs_KeyStore); + NoteDecryptorMap::const_iterator mi = mapNoteDecryptors.find(address); + if (mi != mapNoteDecryptors.end()) + { + decOut = mi->second; + return true; + } + } + return false; + } void GetPaymentAddresses(std::set &setAddress) const { setAddress.clear(); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 9dde25850d0..8cb630ecdc3 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -855,24 +855,25 @@ mapNoteAddrs_t CWallet::FindMyNotes(const CTransaction& tx) const std::set addresses; GetPaymentAddresses(addresses); libzcash::SpendingKey key; + ZCNoteDecryption decryptor; for (size_t i = 0; i < tx.vjoinsplit.size(); i++) { auto hSig = tx.vjoinsplit[i].h_sig(*pzcashParams, tx.joinSplitPubKey); for (size_t j = 0; j < tx.vjoinsplit[i].ciphertexts.size(); j++) { BOOST_FOREACH(const libzcash::PaymentAddress& address, addresses) { - GetSpendingKey(address, key); - ZCNoteDecryption decryptor(key.viewing_key()); - try { - libzcash::NotePlaintext::decrypt( - decryptor, - tx.vjoinsplit[i].ciphertexts[j], - tx.vjoinsplit[i].ephemeralKey, - hSig, - (unsigned char) j); - noteAddrs.insert(std::make_pair(pNoteIndex_t(i, j), address)); - break; - } catch (const std::exception &) { - // Couldn't decrypt with this spending key + if (GetSpendingKey(address, key) && GetNoteDecryptor(address, decryptor)) { + try { + libzcash::NotePlaintext::decrypt( + decryptor, + tx.vjoinsplit[i].ciphertexts[j], + tx.vjoinsplit[i].ephemeralKey, + hSig, + (unsigned char) j); + noteAddrs.insert(std::make_pair(pNoteIndex_t(i, j), address)); + break; + } catch (const std::exception &) { + // Couldn't decrypt with this spending key + } } } } diff --git a/src/zcash/NoteEncryption.hpp b/src/zcash/NoteEncryption.hpp index ac8fb321fb8..ad8fc49f3ce 100644 --- a/src/zcash/NoteEncryption.hpp +++ b/src/zcash/NoteEncryption.hpp @@ -61,6 +61,8 @@ class NoteDecryption { typedef boost::array Ciphertext; typedef boost::array Plaintext; + // Unused default constructor to make allocators happy + NoteDecryption() { } NoteDecryption(uint256 sk_enc); Plaintext decrypt(const Ciphertext &ciphertext, From 2a263db14864a007634f356db6fcdcd8401ba920 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 12 Aug 2016 22:05:37 +1200 Subject: [PATCH 22/22] Keep track of spent Notes, and detect and report conflicts --- src/wallet/wallet.cpp | 53 +++++++++++++++++++++++++++++++++++++++---- src/wallet/wallet.h | 16 +++++++++++-- 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 8cb630ecdc3..dd968c2fd9c 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -396,6 +396,20 @@ set CWallet::GetConflicts(const uint256& txid) const for (TxSpends::const_iterator it = range.first; it != range.second; ++it) result.insert(it->second); } + + std::pair range_n; + + BOOST_FOREACH(const JSDescription& jsdesc, wtx.vjoinsplit) + { + BOOST_FOREACH(const uint256& nullifier, jsdesc.nullifiers) + { + if (mapTxNullifiers.count(nullifier) <= 1) + continue; // No conflict if zero or one spends + range_n = mapTxNullifiers.equal_range(nullifier); + for (TxNullifiers::const_iterator it = range_n.first; it != range_n.second; ++it) + result.insert(it->second); + } + } return result; } @@ -451,7 +465,8 @@ bool CWallet::Verify(const string& walletFile, string& warningString, string& er return true; } -void CWallet::SyncMetaData(pair range) +template +void CWallet::SyncMetaData(pair::iterator, typename TxSpendMap::iterator> range) { // We want all the wallet transactions in range to have the same metadata as // the oldest (smallest nOrderPos). @@ -459,7 +474,7 @@ void CWallet::SyncMetaData(pair range) int nMinOrderPos = std::numeric_limits::max(); const CWalletTx* copyFrom = NULL; - for (TxSpends::iterator it = range.first; it != range.second; ++it) + for (typename TxSpendMap::iterator it = range.first; it != range.second; ++it) { const uint256& hash = it->second; int n = mapWallet[hash].nOrderPos; @@ -470,7 +485,7 @@ void CWallet::SyncMetaData(pair range) } } // Now copy data from copyFrom to rest: - for (TxSpends::iterator it = range.first; it != range.second; ++it) + for (typename TxSpendMap::iterator it = range.first; it != range.second; ++it) { const uint256& hash = it->second; CWalletTx* copyTo = &mapWallet[hash]; @@ -507,15 +522,42 @@ bool CWallet::IsSpent(const uint256& hash, unsigned int n) const return false; } +/** + * Note is spent if any non-conflicted transaction + * spends it: + */ +bool CWallet::IsSpent(const uint256& nullifier) const +{ + pair range; + range = mapTxNullifiers.equal_range(nullifier); + + for (TxNullifiers::const_iterator it = range.first; it != range.second; ++it) + { + const uint256& wtxid = it->second; + std::map::const_iterator mit = mapWallet.find(wtxid); + if (mit != mapWallet.end() && mit->second.GetDepthInMainChain() >= 0) + return true; // Spent + } + return false; +} + void CWallet::AddToSpends(const COutPoint& outpoint, const uint256& wtxid) { mapTxSpends.insert(make_pair(outpoint, wtxid)); pair range; range = mapTxSpends.equal_range(outpoint); - SyncMetaData(range); + SyncMetaData(range); } +void CWallet::AddToSpends(const uint256& nullifier, const uint256& wtxid) +{ + mapTxNullifiers.insert(make_pair(nullifier, wtxid)); + + pair range; + range = mapTxNullifiers.equal_range(nullifier); + SyncMetaData(range); +} void CWallet::AddToSpends(const uint256& wtxid) { @@ -526,6 +568,9 @@ void CWallet::AddToSpends(const uint256& wtxid) BOOST_FOREACH(const CTxIn& txin, thisTx.vin) AddToSpends(txin.prevout, wtxid); + BOOST_FOREACH(const JSDescription& jsdesc, thisTx.vjoinsplit) + BOOST_FOREACH(const uint256& nullifier, jsdesc.nullifiers) + AddToSpends(nullifier, wtxid); } bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index a827be4775b..aafb4eca208 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -472,17 +472,28 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface int64_t nLastResend; bool fBroadcastTransactions; + template + using TxSpendMap = std::multimap; /** * Used to keep track of spent outpoints, and * detect and report conflicts (double-spends or * mutated transactions where the mutant gets mined). */ - typedef std::multimap TxSpends; + typedef TxSpendMap TxSpends; TxSpends mapTxSpends; + /** + * Used to keep track of spent Notes, and + * detect and report conflicts (double-spends). + */ + typedef TxSpendMap TxNullifiers; + TxNullifiers mapTxNullifiers; + void AddToSpends(const COutPoint& outpoint, const uint256& wtxid); + void AddToSpends(const uint256& nullifier, const uint256& wtxid); void AddToSpends(const uint256& wtxid); - void SyncMetaData(std::pair); + template + void SyncMetaData(std::pair::iterator, typename TxSpendMap::iterator>); public: /* @@ -560,6 +571,7 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, std::vector vCoins, std::set >& setCoinsRet, CAmount& nValueRet) const; bool IsSpent(const uint256& hash, unsigned int n) const; + bool IsSpent(const uint256& nullifier) const; bool IsLockedCoin(uint256 hash, unsigned int n) const; void LockCoin(COutPoint& output);