Wallet3 squashed initial commit

tx scanning for basic transactions working
  - TODO: subadresses.  The scanning code is there, but it does not
  currently know/care about any subaddresses.

daemon comms for basic syncing working

(multi-)wallet sync more or less working properly
  - seem to have a dangling shared_ptr somewhere when removing a wallet from
  daemon comms, so not working perfectly yet.

Lots of TODOs and cleanup needed, as well as further features of course.

.clang-format

@@ -0,0 +1,56 @@
+BasedOnStyle: Google
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: 'false'
+AlignConsecutiveDeclarations: 'false'
+AlignEscapedNewlinesLeft: 'true'
+AlignOperands: 'false'
+AlignTrailingComments: 'true'
+AllowShortBlocksOnASingleLine: 'false'
+AllowShortCaseLabelsOnASingleLine: 'false'
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: 'false'
+AllowShortLoopsOnASingleLine: 'false'
+AlwaysBreakAfterDefinitionReturnType: All
+AlwaysBreakAfterReturnType: All
+AlwaysBreakTemplateDeclarations: 'true'
+BreakBeforeBinaryOperators: NonAssignment
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterCaseLabel: true
+  AfterClass: true
+  AfterControlStatement: true
+  AfterEnum: true
+  AfterFunction: true
+  AfterNamespace: true
+  AfterObjCDeclaration: true
+  AfterStruct: true
+  AfterUnion: true
+  AfterExternBlock: true
+  BeforeCatch: true
+  BeforeElse: true
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
+BreakBeforeTernaryOperators: 'true'
+BreakConstructorInitializersBeforeComma: 'true'
+Cpp11BracedListStyle: 'true'
+KeepEmptyLinesAtTheStartOfBlocks: 'false'
+NamespaceIndentation: All
+PenaltyBreakString: '3'
+SpaceBeforeParens: ControlStatements
+SpacesInAngles: 'false'
+SpacesInContainerLiterals: 'false'
+SpacesInParentheses: 'false'
+SpacesInSquareBrackets: 'false'
+Standard: Cpp11
+UseTab: Never
+SortIncludes: false
+ColumnLimit: 100
+
+# treat pointers and reference declarations as if part of the type
+DerivePointerAlignment: false
+PointerAlignment: Left
+
+# when wrapping function calls/declarations, force each parameter to have its own line
+BinPackParameters: 'false'
+BinPackArguments: 'false'

.gitmodules

@@ -31,6 +31,9 @@
 [submodule "external/fmt"]
 	path = external/fmt
 	url = https://github.com/fmtlib/fmt.git
+[submodule "external/Catch2"]
+	path = external/Catch2
+	url = https://github.com/catchorg/Catch2
 [submodule "external/SQLiteCpp"]
 	path = external/SQLiteCpp
 	url = https://github.com/SRombauts/SQLiteCpp
@@ -43,3 +46,6 @@
 [submodule "external/fmt"]
 	path = external/fmt
 	url = https://github.com/fmtlib/fmt.git
+[submodule "external/oxenc"]
+	path = external/oxenc
+	url = https://www.github.com/oxen-io/oxen-encoding.git

CMakeLists.txt

@@ -291,6 +291,7 @@ if(NOT MANUAL_SUBMODULES)
     check_submodule(external/loki-mq cppzmq)
     if(BUILD_TESTS)
       check_submodule(external/googletest)
+      check_submodule(external/Catch2)
     endif()
     check_submodule(external/uWebSockets uSockets)
     check_submodule(external/ghc-filesystem)
@@ -512,7 +513,6 @@ if(NOT BUILD_STATIC_DEPS)
 endif()
 
 add_subdirectory(external)
-
 target_compile_definitions(easylogging PRIVATE AUTO_INITIALIZE_EASYLOGGINGPP)
 
 if(USE_DEVICE_TREZOR)

contrib/format.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+CLANG_FORMAT_DESIRED_VERSION=11
+
+TARGET_DIRS=(src/wallet3 src/sqlitedb)
+
+binary=$(which clang-format-$CLANG_FORMAT_DESIRED_VERSION 2>/dev/null)
+if [ $? -ne 0 ]; then
+    binary=$(which clang-format-mp-$CLANG_FORMAT_DESIRED_VERSION 2>/dev/null)
+fi
+if [ $? -ne 0 ]; then
+    binary=$(which clang-format 2>/dev/null)
+    if [ $? -ne 0 ]; then
+        echo "Please install clang-format version $CLANG_FORMAT_DESIRED_VERSION and re-run this script."
+        exit 1
+    fi
+    version=$(clang-format --version)
+    if [[ ! $version == *"clang-format version $CLANG_FORMAT_DESIRED_VERSION"* ]]; then
+        echo "Please install clang-format version $CLANG_FORMAT_DESIRED_VERSION and re-run this script."
+        exit 1
+    fi
+fi
+
+cd "$(dirname $0)/../"
+if [ "$1" = "verify" ] ; then
+    for d in ${TARGET_DIRS[@]}; do
+        if [ $($binary --output-replacements-xml $(find $d | grep -E '\.([hc](pp)?|mm?)$' | grep -v '\#') | grep '</replacement>' | wc -l) -ne 0 ] ; then
+            exit 1
+        fi
+    done
+else
+    for d in ${TARGET_DIRS[@]}; do
+        echo "Formatting $d"
+        $binary -i $(find $d | grep -E '\.([hc](pp)?|mm)$' | grep -v '\#') &> /dev/null
+    done
+fi
+
+swift_format=$(which swiftformat 2>/dev/null)
+if [ $? -eq 0 ]; then
+    if [ "$1" = "verify" ] ; then
+        for f in $(find daemon | grep -E '\.swift$' | grep -v '\#') ; do
+            if [ $($swift_format --quiet --dryrun < "$f" | diff "$f" - | wc -l) -ne 0 ] ; then
+                exit 1
+            fi
+        done
+    else
+        $swift_format --quiet $(find daemon | grep -E '\.swift$' | grep -v '\#')
+    fi
+
+fi

external/CMakeLists.txt

@@ -80,6 +80,8 @@ else()
   message(STATUS "Found liboxenmq ${OXENMQ_VERSION}")
 endif()
 
+add_subdirectory(oxenc)
+
 add_subdirectory(db_drivers)
 add_subdirectory(easylogging++ easyloggingpp)
 add_subdirectory(randomx EXCLUDE_FROM_ALL)
@@ -168,3 +170,5 @@ set(SQLITECPP_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
 set(SQLITECPP_BUILD_TESTS OFF CACHE BOOL "" FORCE)
 
 add_subdirectory(SQLiteCpp)
+
+add_subdirectory(Catch2)

src/CMakeLists.txt

@@ -62,6 +62,7 @@ add_subdirectory(multisig)
 add_subdirectory(net)
 add_subdirectory(mnemonics)
 add_subdirectory(wallet)
+add_subdirectory(wallet3)
 add_subdirectory(cryptonote_protocol)
 add_subdirectory(blockchain_db)
 add_subdirectory(rpc)

src/crypto/crypto.cpp

@@ -198,6 +198,13 @@ namespace crypto {
     return true;
   }
 
+  crypto::key_derivation generate_key_derivation(const public_key &key1, const secret_key &key2) {
+    //TODO: replace the bool return type version of this function entirely
+    crypto::key_derivation d;
+    generate_key_derivation(key1, key2, d);
+    return d;
+  }
+
   bool generate_key_derivation(const public_key &key1, const secret_key &key2, key_derivation &derivation) {
     ge_p3 point;
     ge_p2 point2;

src/crypto/crypto.h

@@ -196,6 +196,7 @@ namespace crypto {
    * * The sender uses key derivation and the receivers' "spend" key to derive an ephemeral public key.
    * * The receiver can either derive the public key (to check that the transaction is addressed to him) or the private key (to spend the money).
    */
+  crypto::key_derivation generate_key_derivation(const public_key &key1, const secret_key &key2);
   bool generate_key_derivation(const public_key &key1, const secret_key &key2, key_derivation &derivation);
   bool derive_public_key(const key_derivation &derivation, std::size_t output_index, const public_key &base, public_key &derived_key);
   void derivation_to_scalar(const key_derivation &derivation, size_t output_index, ec_scalar &res);

src/cryptonote_basic/cryptonote_basic.cpp

@@ -1,5 +1,7 @@
 #include "cryptonote_basic.h"
 
+#include "cryptonote_format_utils.h"
+
 namespace cryptonote {
 
 void transaction_prefix::set_null() {
@@ -12,6 +14,26 @@ void transaction_prefix::set_null() {
   type = txtype::standard;
 }
 
+std::vector<crypto::public_key> transaction_prefix::get_public_keys() const
+{
+  std::vector<cryptonote::tx_extra_field> fields;
+
+  if (!parse_tx_extra(extra, fields))
+  {
+    throw std::invalid_argument("Failed to parse tx_extra of a transaction.");
+  }
+
+  std::vector<crypto::public_key> keys;
+  tx_extra_pub_key pk_field;
+  size_t i=0;
+  while(find_tx_extra_field_by_type(fields, pk_field, i++))
+  {
+    keys.push_back(pk_field.pub_key);
+  }
+
+  return keys;
+}
+
 transaction::transaction(const transaction &t) :
   transaction_prefix(t),
   hash_valid(false),

src/cryptonote_basic/cryptonote_basic.h

@@ -232,6 +232,7 @@ namespace cryptonote
       return unlock_time;
     }
 
+    std::vector<crypto::public_key> get_public_keys() const;
   };
 
   class transaction final : public transaction_prefix

src/device/device.hpp

@@ -173,6 +173,7 @@ namespace hw {
         virtual bool  scalarmultBase(rct::key &aG, const rct::key &a) = 0;
         virtual bool  sc_secret_add( crypto::secret_key &r, const crypto::secret_key &a, const crypto::secret_key &b) = 0;
         virtual crypto::secret_key  generate_keys(crypto::public_key &pub, crypto::secret_key &sec, const crypto::secret_key& recovery_key = crypto::secret_key(), bool recover = false) = 0;
+        virtual crypto::key_derivation  generate_key_derivation(const crypto::public_key &pub, const crypto::secret_key &sec) = 0;
         virtual bool  generate_key_derivation(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_derivation &derivation) = 0;
         virtual bool  conceal_derivation(crypto::key_derivation &derivation, const crypto::public_key &tx_pub_key, const std::vector<crypto::public_key> &additional_tx_pub_keys, const crypto::key_derivation &main_derivation, const std::vector<crypto::key_derivation> &additional_derivations) = 0;
         virtual bool  derivation_to_scalar(const crypto::key_derivation &derivation, const size_t output_index, crypto::ec_scalar &res) = 0;

src/device/device_default.cpp

@@ -237,6 +237,11 @@ namespace hw {
             return crypto::generate_keys(pub, sec, recovery_key, recover);
         }
 
+        crypto::key_derivation device_default::generate_key_derivation(const crypto::public_key &pub, const crypto::secret_key &sec)
+        {
+            return crypto::generate_key_derivation(pub, sec);
+        }
+
         bool device_default::generate_key_derivation(const crypto::public_key &key1, const crypto::secret_key &key2, crypto::key_derivation &derivation) {
             return crypto::generate_key_derivation(key1, key2, derivation);
         }

src/device/device_default.hpp

@@ -94,6 +94,7 @@ namespace hw {
             bool  scalarmultBase(rct::key &aG, const rct::key &a) override;
             bool  sc_secret_add(crypto::secret_key &r, const crypto::secret_key &a, const crypto::secret_key &b) override;
             crypto::secret_key  generate_keys(crypto::public_key &pub, crypto::secret_key &sec, const crypto::secret_key& recovery_key = crypto::secret_key(), bool recover = false) override;
+            crypto::key_derivation generate_key_derivation(const crypto::public_key &pub, const crypto::secret_key &sec) override;
             bool  generate_key_derivation(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_derivation &derivation) override;
             bool  conceal_derivation(crypto::key_derivation &derivation, const crypto::public_key &tx_pub_key, const std::vector<crypto::public_key> &additional_tx_pub_keys, const crypto::key_derivation &main_derivation, const std::vector<crypto::key_derivation> &additional_derivations) override;
             bool  derivation_to_scalar(const crypto::key_derivation &derivation, const size_t output_index, crypto::ec_scalar &res) override;

src/device/device_ledger.cpp

@@ -1033,6 +1033,13 @@ namespace hw {
 
     }
 
+    crypto::key_derivation device_ledger::generate_key_derivation(const crypto::public_key &pub, const crypto::secret_key &sec) {
+      //TODO: replace entirely the bool return version of this
+      crypto::key_derivation d;
+      generate_key_derivation(pub, sec, d);
+      return d;
+    }
+
     bool device_ledger::generate_key_derivation(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_derivation &derivation) {
         auto locks = tools::unique_locks(device_locker, command_locker);
         bool r = false;

src/device/device_ledger.hpp

@@ -275,6 +275,7 @@ namespace hw {
         bool  scalarmultBase(rct::key &aG, const rct::key &a) override;
         bool  sc_secret_add(crypto::secret_key &r, const crypto::secret_key &a, const crypto::secret_key &b) override;
         crypto::secret_key  generate_keys(crypto::public_key &pub, crypto::secret_key &sec, const crypto::secret_key& recovery_key = crypto::secret_key(), bool recover = false) override;
+        crypto::key_derivation  generate_key_derivation(const crypto::public_key &pub, const crypto::secret_key &sec) override;
         bool  generate_key_derivation(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_derivation &derivation) override;
         bool  conceal_derivation(crypto::key_derivation &derivation, const crypto::public_key &tx_pub_key, const std::vector<crypto::public_key> &additional_tx_pub_keys, const crypto::key_derivation &main_derivation, const std::vector<crypto::key_derivation> &additional_derivations) override;
         bool  derivation_to_scalar(const crypto::key_derivation &derivation, const size_t output_index, crypto::ec_scalar &res) override;

src/rpc/CMakeLists.txt

@@ -84,6 +84,7 @@ target_link_libraries(rpc
 
 target_link_libraries(daemon_rpc_server
   PRIVATE
+    oxenc
     rpc_server_base
     rpc
     Boost::thread