Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

UTXO merging #1937

Merged
merged 6 commits into from Aug 25, 2022
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions src/CMakeLists.txt
Expand Up @@ -117,6 +117,7 @@ add_executable(${PROJECT_NAME}_tests MACOSX_BUNDLE ${ICON}
tests/api/komodo_prices/komodo.prices.tests.cpp
tests/api/mm2/mm2.rpc.trade.preimage.tests.cpp
tests/api/mm2/mm2.fraction.tests.cpp
tests/api/mm2/mm2.api.utxo.merge.params.tests.cpp

##! Utilities
tests/utilities/qt.utilities.tests.cpp
Expand Down
4 changes: 3 additions & 1 deletion src/core/atomicdex/api/mm2/rpc.electrum.cpp
Expand Up @@ -38,7 +38,9 @@ namespace mm2::api
if (cfg.address_format.has_value()) {
j["address_format"] = cfg.address_format.value();
}
//SPDLOG_INFO("electrum: {}", j.dump());
if (cfg.merge_params.has_value()) {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Allman style indentation

j["utxo_merge_params"] = cfg.merge_params.value();
}
}

//! Deserialization
Expand Down
21 changes: 11 additions & 10 deletions src/core/atomicdex/api/mm2/rpc.electrum.hpp
Expand Up @@ -27,16 +27,17 @@ namespace mm2::api
{
struct electrum_request
{
std::string coin_name;
std::vector<atomic_dex::electrum_server> servers;
CoinType coin_type;
bool is_testnet{false};
bool with_tx_history{true};
const std::string testnet_qrc_swap_contract_address{"0xba8b71f3544b93e2f681f996da519a98ace0107a"};
const std::string testnet_fallback_qrc_swap_contract_address{testnet_qrc_swap_contract_address};
const std::string mainnet_qrc_swap_contract_address{"0x2f754733acd6d753731c00fee32cb484551cc15d"};
const std::string mainnet_fallback_qrc_swap_contract_address{mainnet_qrc_swap_contract_address};
std::optional<nlohmann::json> address_format;
std::string coin_name;
std::vector<atomic_dex::electrum_server> servers;
CoinType coin_type;
bool is_testnet{false};
bool with_tx_history{true};
const std::string testnet_qrc_swap_contract_address{"0xba8b71f3544b93e2f681f996da519a98ace0107a"};
const std::string testnet_fallback_qrc_swap_contract_address{testnet_qrc_swap_contract_address};
const std::string mainnet_qrc_swap_contract_address{"0x2f754733acd6d753731c00fee32cb484551cc15d"};
const std::string mainnet_fallback_qrc_swap_contract_address{mainnet_qrc_swap_contract_address};
std::optional<nlohmann::json> address_format;
std::optional<nlohmann::json> merge_params;
};

struct electrum_answer
Expand Down
32 changes: 32 additions & 0 deletions src/core/atomicdex/api/mm2/utxo.merge.params.cpp
@@ -0,0 +1,32 @@
/******************************************************************************
* Copyright © 2013-2022 The Komodo Platform Developers. *
* *
* See the AUTHORS, DEVELOPER-AGREEMENT and LICENSE files at *
* the top-level directory of this distribution for the individual copyright *
* holder information and the developer policies on copyright and licensing. *
* *
* Unless otherwise agreed in a custom licensing agreement, no part of the *
* Komodo Platform software, including this file may be copied, modified, *
* propagated or distributed except according to the terms contained in the *
* LICENSE file *
* *
* Removal or modification of this copyright notice is prohibited. *
* *
******************************************************************************/

//! Dependencies Headers
#include <nlohmann/json.hpp>

//! Project Headers
#include "utxo.merge.params.hpp"

namespace mm2::api
{
void
to_json(nlohmann::json& j, const utxo_merge_params& cfg)
{
j["merge_at"] = cfg.merge_at;
j["check_every"] = cfg.check_every;
j["max_merge_at_once"] = cfg.max_merge_at_once;
}
} // namespace mm2::api
36 changes: 36 additions & 0 deletions src/core/atomicdex/api/mm2/utxo.merge.params.hpp
@@ -0,0 +1,36 @@
/******************************************************************************
* Copyright © 2013-2022 The Komodo Platform Developers. *
* *
* See the AUTHORS, DEVELOPER-AGREEMENT and LICENSE files at *
* the top-level directory of this distribution for the individual copyright *
* holder information and the developer policies on copyright and licensing. *
* *
* Unless otherwise agreed in a custom licensing agreement, no part of the *
* Komodo Platform software, including this file may be copied, modified, *
* propagated or distributed except according to the terms contained in the *
* LICENSE file *
* *
* Removal or modification of this copyright notice is prohibited. *
* *
******************************************************************************/

#pragma once

#include <nlohmann/json_fwd.hpp>

namespace mm2::api
{
struct utxo_merge_params
{
std::size_t merge_at;
std::size_t check_every;
std::size_t max_merge_at_once;
};

void to_json(nlohmann::json& j, const utxo_merge_params& cfg);
}

namespace atomic_dex
{
using t_utxo_merge_params = ::mm2::api::utxo_merge_params;
} // namespace atomic_dex
5 changes: 5 additions & 0 deletions src/core/atomicdex/config/coins.cfg.cpp
Expand Up @@ -33,6 +33,11 @@ namespace atomic_dex
{
cfg.custom_backup = j.at("mm2_backup");
}

if (j.contains("utxo_merge"))
{
cfg.utxo_merge = j.at("utxo_merge");
}
if (j.contains("electrum"))
{
cfg.electrum_urls = j.at("electrum").get<std::vector<electrum_server>>();
Expand Down
58 changes: 30 additions & 28 deletions src/core/atomicdex/config/coins.cfg.hpp
Expand Up @@ -27,6 +27,7 @@

//! Project
#include "atomicdex/api/mm2/mm2.constants.hpp"
#include "atomicdex/api/mm2/utxo.merge.params.hpp"
#include "atomicdex/config/electrum.cfg.hpp"
#include "atomicdex/constants/qt.coins.enums.hpp"

Expand All @@ -39,34 +40,35 @@ namespace atomic_dex
static constexpr const char* matic_gas_stations = "https://gasstation-mainnet.matic.network/";
using electrum_servers = std::vector<electrum_server>;
using nodes = std::vector<std::string>;
std::string ticker;
std::optional<std::string> alias_ticker{std::nullopt};
std::string gui_ticker; ///< Ticker displayed in the gui
std::string name; ///< nice name
std::optional<electrum_servers> electrum_urls;
std::optional<nodes> urls;
bool is_claimable{false};
std::string minimal_claim_amount{"0"};
bool currently_enabled{false};
bool active{false};
std::string coinpaprika_id{"test-coin"};
std::string coingecko_id{"test-coin"};
std::string nomics_id{"test-coin"};
bool is_custom_coin{false};
std::string type;
std::vector<std::string> explorer_url; ///< usefull for transaction, take this url and append transaction id
std::string tx_uri{"tx/"};
std::string address_url{"address/"};
std::optional<nlohmann::json> custom_backup;
std::optional<bool> is_testnet{false}; ///< True if testnet (tBTC, tQTUM, QRC-20 on testnet, tETH)
CoinType coin_type;
bool checked{false};
bool wallet_only{false};
bool has_parent_fees_ticker{false}; ///< True if parent fees is different from current ticker eg: ERC20 tokens
std::string fees_ticker;
bool segwit{false};
bool is_segwit_on{false};
bool is_erc_family{false};
std::string ticker;
std::optional<std::string> alias_ticker{std::nullopt};
std::string gui_ticker; ///< Ticker displayed in the gui
std::string name; ///< nice name
std::optional<electrum_servers> electrum_urls;
std::optional<bool> utxo_merge{false};
std::optional<nodes> urls;
bool is_claimable{false};
std::string minimal_claim_amount{"0"};
bool currently_enabled{false};
bool active{false};
std::string coinpaprika_id{"test-coin"};
std::string coingecko_id{"test-coin"};
std::string nomics_id{"test-coin"};
bool is_custom_coin{false};
std::string type;
std::vector<std::string> explorer_url; ///< usefull for transaction, take this url and append transaction id
std::string tx_uri{"tx/"};
std::string address_url{"address/"};
std::optional<nlohmann::json> custom_backup;
std::optional<bool> is_testnet{false}; ///< True if testnet (tBTC, tQTUM, QRC-20 on testnet, tETH)
CoinType coin_type;
bool checked{false};
bool wallet_only{false};
bool has_parent_fees_ticker{false}; ///< True if parent fees is different from current ticker eg: ERC20 tokens
std::string fees_ticker;
bool segwit{false};
bool is_segwit_on{false};
bool is_erc_family{false};
};

void from_json(const nlohmann::json& j, coin_config& cfg);
Expand Down
9 changes: 9 additions & 0 deletions src/core/atomicdex/services/mm2/mm2.service.cpp
Expand Up @@ -641,8 +641,17 @@ namespace atomic_dex
request.address_format = nlohmann::json::object();
request.address_format.value()["format"] = "segwit";
}

if (coin_info.utxo_merge.value_or(false))
{
t_utxo_merge_params params{.merge_at = 300, .check_every = 60, .max_merge_at_once = 200};
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we would have to test if it works with max_merge_at_once=200, i can imagine it would cause issues with some electrums, so maybe it's better to set max_merge_at_once to 100
check_every = 60 means it checks every minute... do we need to do this every minute? maybe better to do it every 10 minutes or so
any maybe also better to start earlier, not wait for 300 utxos
what do you guys think?

Copy link
Collaborator Author

@smk762 smk762 Aug 15, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I"m happy to tweak the default params - just used those to drop around 3500 RICK utxos into 130 or so relatively quickly.

There is a user with 23000 utxos from mining tokel, which will take a while to consolidate either way.
23000 @ 200 utxos every 10 mins = around 19hrs to consolidate.

From memory, up to 800 or so utxos can be merged in a tx.
I"m not sure at what number utxos electrums begin to complain about "payload too large"
I think every 5 min is a balance between not too obtrusive, and not too slow. Every minute could cause clogging issues when blocks are slow.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, sounds like it will work... there is a difference though, RICK (and all other coins run by me) electrums have MAX_SEND = 2000000 set, the default is 1M

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updated to use these values:
merge_at = 250
check_every = 300
max_merge_at_once = 125

nlohmann::json j;
mm2::api::to_json(j, params);
request.merge_params = j;
}
nlohmann::json j = ::mm2::api::template_request("electrum");
::mm2::api::to_json(j, request);
// SPDLOG_INFO("electrum request: {}", j.dump(4));
This conversation was marked as resolved.
Show resolved Hide resolved
batch_array.push_back(j);
}
else
Expand Down
36 changes: 36 additions & 0 deletions src/tests/api/mm2/mm2.api.utxo.merge.params.tests.cpp
@@ -0,0 +1,36 @@
/******************************************************************************
* Copyright © 2013-2022 The Komodo Platform Developers. *
* *
* See the AUTHORS, DEVELOPER-AGREEMENT and LICENSE files at *
* the top-level directory of this distribution for the individual copyright *
* holder information and the developer policies on copyright and licensing. *
* *
* Unless otherwise agreed in a custom licensing agreement, no part of the *
* Komodo Platform software, including this file may be copied, modified, *
* propagated or distributed except according to the terms contained in the *
* LICENSE file *
* *
* Removal or modification of this copyright notice is prohibited. *
* *
******************************************************************************/

//! Deps
#include "doctest/doctest.h"
#include <nlohmann/json.hpp>

#include "atomicdex/api/mm2/utxo.merge.params.hpp"

TEST_CASE("mm2::api::utxo_merge_params serialisation")
{
const nlohmann::json expected_json = R"(
{
"merge_at":50,
"check_every":10,
"max_merge_at_once":25
}
)"_json;
mm2::api::utxo_merge_params request{.merge_at = 50, .check_every = 10, .max_merge_at_once = 25};
nlohmann::json j;
mm2::api::to_json(j, request);
CHECK_EQ(j, expected_json);
}