Feature: Add full edit transaction for RBF (#998)

* Add full edit transaction for RBF

* docs: update TOC

Co-authored-by: Kim Neunert <kneunert@gmail.com>
Co-authored-by: k9ert <k9ert@users.noreply.github.com>

DEVELOPMENT.md

@@ -4,13 +4,15 @@
 
 - [Development](#development)
   - [How to run the Application](#how-to-run-the-application)
-  - [Howto run the tests](#howto-run-the-tests)
+  - [How to run the tests](#how-to-run-the-tests)
   - [Code-Style](#code-style)
   - [Developing on tests](#developing-on-tests)
     - [bitcoin-specific stuff](#bitcoin-specific-stuff)
     - [Cypress UI-testing](#cypress-ui-testing)
   - [Flask specific stuff](#flask-specific-stuff)
   - [More on the bitcoind requirements](#more-on-the-bitcoind-requirements)
+    - [Automatically mine and deposit test coins](#automatically-mine-and-deposit-test-coins)
+    - [Manually mine and deposit test coins](#manually-mine-and-deposit-test-coins)
   - [IDE-specific Configuration (might be outdated)](#ide-specific-configuration-might-be-outdated)
     - [Visual Studio Code](#visual-studio-code)
       - [Debugging](#debugging)

src/cryptoadvance/specter/server_endpoints/wallets.py

@@ -571,9 +571,12 @@ def send_new(wallet_alias):
     fee_rate = 0.0
     fee_rate_blocks = 6
     rbf = True
+    rbf_utxo = []
+    rbf_tx_id = ""
     selected_coins = request.form.getlist("coinselect")
     if request.method == "POST":
         action = request.form.get("action")
+        rbf_tx_id = request.form.get("rbf_tx_id", "")
         if action == "createpsbt":
             i = 0
             addresses = []
@@ -635,6 +638,7 @@ def send_new(wallet_alias):
                     rbf=rbf,
                     selected_coins=selected_coins,
                     readonly="estimate_fee" in request.form,
+                    rbf_edit_mode=(rbf_tx_id != ""),
                 )
                 if psbt is None:
                     err = "Probably you don't have enough funds, or something else..."
@@ -666,7 +670,7 @@ def send_new(wallet_alias):
             try:
                 rbf_tx_id = request.form["rbf_tx_id"]
                 rbf_fee_rate = float(request.form["rbf_fee_rate"])
-                psbt = wallet.send_rbf_tx(rbf_tx_id, rbf_fee_rate)
+                psbt = wallet.bumpfee(rbf_tx_id, rbf_fee_rate)
                 return render_template(
                     "wallet/send/sign/wallet_send_sign_psbt.jinja",
                     psbt=psbt,
@@ -678,6 +682,20 @@ def send_new(wallet_alias):
                 )
             except Exception as e:
                 flash("Failed to perform RBF. Error: %s" % e, "error")
+        elif action == "rbf_edit":
+            try:
+                decoded_tx = wallet.decode_tx(rbf_tx_id)
+                addresses = decoded_tx["addresses"]
+                amounts = decoded_tx["amounts"]
+                selected_coins = [
+                    f"{utxo['txid']}, {utxo['vout']}"
+                    for utxo in decoded_tx["used_utxo"]
+                ]
+                fee_rate = float(request.form["rbf_fee_rate"])
+                fee_options = "manual"
+                rbf = True
+            except Exception as e:
+                flash("Failed to perform RBF. Error: %s" % e, "error")
         elif action == "signhotwallet":
             passphrase = request.form["passphrase"]
             psbt = ast.literal_eval(request.form["psbt"])
@@ -714,6 +732,13 @@ def send_new(wallet_alias):
                 specter=app.specter,
                 rand=rand,
             )
+
+    if rbf_tx_id:
+        try:
+            rbf_utxo = wallet.get_rbf_utxo(rbf_tx_id)
+        except Exception as e:
+            flash("Failed to get RBF coins. Error: %s" % e, "error")
+
     show_advanced_settings = (
         ui_option != "ui"
         or subtract
@@ -736,6 +761,8 @@ def send_new(wallet_alias):
         rbf=rbf,
         selected_coins=selected_coins,
         show_advanced_settings=show_advanced_settings,
+        rbf_utxo=rbf_utxo,
+        rbf_tx_id=rbf_tx_id,
         wallet_alias=wallet_alias,
         wallet=wallet,
         specter=app.specter,

src/cryptoadvance/specter/templates/device/new_device.jinja

@@ -624,12 +624,12 @@
 
         function toggleAdvancedXpubs() {
             let advancedButton = document.getElementById('toggle_advanced_xpubs');
-            let advancedSettigns = document.getElementById('advanced_settings_xpubs');
-            if (advancedSettigns.style.display === 'block') {
-                advancedSettigns.style.display = 'none';
+            let advancedSettings = document.getElementById('advanced_settings_xpubs');
+            if (advancedSettings.style.display === 'block') {
+                advancedSettings.style.display = 'none';
                 advancedButton.innerHTML = 'Advanced ▶';
             } else {
-                advancedSettigns.style.display = 'block';
+                advancedSettings.style.display = 'block';
                 advancedButton.innerHTML = 'Advanced ▼';
                 if (totalSteps == 3) {
                     document.getElementById('advanced-hot-wallet').display = 'block';

src/cryptoadvance/specter/templates/device/new_device_manual.jinja

@@ -183,15 +183,15 @@ m/48h/{{ 0 if specter.info.chain == "main" else 1 }}h/0h/2h</textarea>
 <script type="text/javascript">
 	function toggleAdvanced() {
 		let advancedButton = document.getElementById('toggle_advanced');
-		let advancedSettigns = document.getElementById('advanced_settings');
-		if (advancedSettigns.style.display === 'block') {
-			advancedSettigns.style.display = 'none';
+		let advancedSettings = document.getElementById('advanced_settings');
+		if (advancedSettings.style.display === 'block') {
+			advancedSettings.style.display = 'none';
 			advancedButton.innerHTML = 'Advanced &#9654;';
 			if (isCoinSelectionActive()) {
 				toggleExpand();
 			}
 		} else {
-			advancedSettigns.style.display = 'block';
+			advancedSettings.style.display = 'block';
 			advancedButton.innerHTML = 'Advanced &#9660;';
 		}
 	}

src/cryptoadvance/specter/templates/includes/tx-row.html

@@ -131,7 +131,7 @@
                 this.confirmations.innerHTML = `Unconfirmed`;
             }
 
-            if (this.tx.category == "send" && this.tx["bip125-replaceable"] == "yes") {
+            if ((this.tx.category == "send" || this.tx.category == "selftransfer") && this.tx["bip125-replaceable"] == "yes") {
                 this.rbf.classList.remove('hidden');
                 this.rbf.onclick = () => {
                     let txDataPopup = document.getElementById('tx-popup');
@@ -151,8 +151,26 @@ <h1>Speed up the Transaction</h1>
                     <br>
                     <br>
                     <button type="submit" name="action" value="rbf" class="btn centered">Speed up!</button>
+                    <br>
+			        <span class="toggle_advanced_rbf" style="cursor: pointer;">Advanced {% if show_advanced_settings %}&#9660;{% else %}&#9654;{% endif %}</span>
+                    <div class="advanced_rbf hidden warning">
+                        <p style="max-width: 400px;">If you would like further customization, you can click here to fully edit the transaction.<br>(advanced, not recommended for new users)</p>
+                        <button type="submit" name="action" value="rbf_edit" class="btn centered">Edit the transaction (advanced)</button>
+                    </div>
                     </form>
                     `;
+                    txDataPopup.querySelector('.toggle_advanced_rbf').onclick = () => {
+                        let advancedButton = txDataPopup.querySelector('.toggle_advanced_rbf')
+                        let advancedSettings = txDataPopup.querySelector('.advanced_rbf')
+                        if (advancedSettings.classList.contains('hidden')) {
+                            advancedSettings.classList.remove('hidden')
+                            advancedButton.innerHTML = 'Advanced &#9660;';
+                        } else {
+                            advancedSettings.classList.add('hidden')
+                            advancedButton.innerHTML = 'Advanced &#9654;';
+                        }
+                    }
+
                     showPageOverlay('tx-popup');
                 }
             } else {

src/cryptoadvance/specter/templates/wallet/send/new/components/coin_selection.jinja

@@ -1,6 +1,6 @@
 <div>
     {% from 'wallet/send/new/components/coin_selection_table.jinja' import coin_selection_table %}
-    {{ coin_selection_table(wallet.utxo, specter.explorer, selected_coins) }}
+    {{ coin_selection_table(wallet.utxo + rbf_utxo, specter.explorer, selected_coins, rbf_tx_id) }}
     <br>
 </div>
 
@@ -22,8 +22,8 @@
         validateForm();
     }
 
-    function updateCoinSelect(coin) {
-        let coinAmount = parseFloat(coin.value.split(', ')[2]);
+    function updateCoinSelect(coin, amount) {
+        let coinAmount = parseFloat(amount);
         if (coin.checked) {
             coinSelectAmount += coinAmount;
         } else {

src/cryptoadvance/specter/templates/wallet/send/new/components/coin_selection_item.jinja

@@ -12,11 +12,11 @@
     {% from 'wallet/components/explorer_link.jinja' import explorer_link %}
     <tr>
         <td>
-            <input id="coin_{{ [txid, vout, '%.8f'|format(amount)] | join(', ') }}" class="checkbox coin_select_checkbox" type="checkbox" name="coinselect" value="{{ [txid, vout, '%.8f'|format(amount)] | join(', ') }}" onchange='updateCoinSelect(this)' {% if selected %}checked{% endif %}>
+            <input id="coin_{{ [txid, vout] | join(', ') }}" class="checkbox coin_select_checkbox" type="checkbox" name="coinselect" value="{{ [txid, vout] | join(', ') }}" onchange='updateCoinSelect(this, "{{ amount }}")' {% if selected %}checked{% endif %}>
             {% if selected %}
                 <script>
                     document.addEventListener("DOMContentLoaded", function(){
-                        updateCoinSelect(document.getElementById("coin_{{ [txid, vout, '%.8f'|format(amount)] | join(', ') }}"))
+                        updateCoinSelect(document.getElementById("coin_{{ [txid, vout] | join(', ') }}"), "{{ amount }}")
                     });
                 </script>
             {% endif %}

src/cryptoadvance/specter/templates/wallet/send/new/components/coin_selection_table.jinja

@@ -6,18 +6,19 @@
     - unspents: List of wallet UTXOs
     - explorer: explorer link
     - selected_coins: List of UXTOs previously selected (if any)
+    - rbf_tx_id: tx id to skip because the editor is used to edit it in RBF (empty string if not RBF editing)
  #}
-{% macro coin_selection_table(unspents, explorer, selected_coins) -%}
+{% macro coin_selection_table(unspents, explorer, selected_coins, rbf_tx_id) -%}
     <table style="table-layout: fixed; display: {% if selected_coins %}block{% else %}none{% endif %}; max-width:98%;" id="coin_selection_table">
         <thead>
         <tr>
             <th></th><th>TxID</th><th>Address</th><th>Amount</th>
         </tr>
         </thead>
         <tbody>
-            {% for tx in unspents %}
+            {% for tx in unspents if tx['txid'] != rbf_tx_id %}
                 {% from 'wallet/send/new/components/coin_selection_item.jinja' import coin_selection_item %}
-                {{ coin_selection_item(tx['txid'], tx['vout'], tx['amount'], tx['address'], tx['label'], explorer, "{}, {}, {}".format(tx['txid'], tx['vout'], "%.8f"|format(tx['amount'])) in selected_coins) }}
+                {{ coin_selection_item(tx['txid'], tx['vout'], tx['amount'], tx['address'], tx['label'], explorer, "{}, {}".format(tx['txid'], tx['vout']) in selected_coins) }}
             {% endfor %}
         </tbody>
     </table>

src/cryptoadvance/specter/templates/wallet/send/new/wallet_send.jinja

@@ -25,6 +25,7 @@
 
 	<form action="{{ url_for('wallets_endpoint.send_new',wallet_alias=wallet_alias) }}" id="send-form" method="POST" style="width: 100%;">
 		<input type="hidden" class="csrf-token" name="csrf_token" value="{{ csrf_token() }}"/>
+		<input type="hidden" name="rbf_tx_id" value="{{ rbf_tx_id }}"/>
 		<h1 class="padded">Create Transaction</h1>
 		<p class="center">Available Funds: {{wallet.full_available_balance | btcunitamount}}
 			{% if specter.unit == 'sat' %}
@@ -253,6 +254,10 @@
 		}
 
 		function isAboveWalletBalance(unit, amount) {
+			// TODO: Currently check is disabled for RBF for simplicity, should add it back for RBF
+			if ("{{rbf_tx_id}}") {
+				return false;
+			}
 			return (unit == 'sat' ? amount / 1e8 : parseFloat(amount.toFixed(8))) > parseFloat(parseFloat('{{ wallet.full_available_balance }}').toFixed(8));
 		}
 
@@ -400,15 +405,15 @@
 
 		function toggleAdvanced() {
 			let advancedButton = document.getElementById('toggle_advanced');
-			let advancedSettigns = document.getElementById('advanced_settings');
-			if (advancedSettigns.style.display === 'block') {
-				advancedSettigns.style.display = 'none';
+			let advancedSettings = document.getElementById('advanced_settings');
+			if (advancedSettings.style.display === 'block') {
+				advancedSettings.style.display = 'none';
 				advancedButton.innerHTML = 'Advanced &#9654;';
 				if (isCoinSelectionActive()) {
 					toggleExpand();
 				}
 			} else {
-				advancedSettigns.style.display = 'block';
+				advancedSettings.style.display = 'block';
 				advancedButton.innerHTML = 'Advanced &#9660;';
 			}
 		}

src/cryptoadvance/specter/wallet.py

@@ -1238,6 +1238,7 @@ def createpsbt(
         readonly=False,
         rbf=True,
         existing_psbt=None,
+        rbf_edit_mode=False,
     ):
         """
         fee_rate: in sat/B or BTC/kB. If set to 0 Bitcoin Core sets feeRate automatically.
@@ -1249,17 +1250,20 @@ def createpsbt(
         extra_inputs = []
 
         if not existing_psbt:
-            if self.full_available_balance < sum(amounts):
-                raise SpecterError(
-                    "The wallet does not have sufficient funds to make the transaction."
-                )
+            if not rbf_edit_mode:
+                if self.full_available_balance < sum(amounts):
+                    raise SpecterError(
+                        "The wallet does not have sufficient funds to make the transaction."
+                    )
 
             if selected_coins != []:
                 still_needed = sum(amounts)
                 for coin in selected_coins:
                     coin_txid = coin.split(",")[0]
                     coin_vout = int(coin.split(",")[1])
-                    coin_amount = float(coin.split(",")[2])
+                    coin_amount = self.gettransaction(coin_txid, decode=True)["vout"][
+                        coin_vout
+                    ]["value"]
                     extra_inputs.append({"txid": coin_txid, "vout": coin_vout})
                     still_needed -= coin_amount
                     if still_needed < 0:
@@ -1360,7 +1364,63 @@ def createpsbt(
 
         return psbt
 
-    def send_rbf_tx(self, txid, fee_rate):
+    def get_rbf_utxo(self, rbf_tx_id):
+        decoded_tx = self.decode_tx(rbf_tx_id)
+        selected_coins = [
+            f"{utxo['txid']}, {utxo['vout']}" for utxo in decoded_tx["used_utxo"]
+        ]
+        rbf_utxo = [
+            {
+                "txid": tx["txid"],
+                "vout": tx["vout"],
+                "details": self.gettransaction(tx["txid"], decode=True)["vout"][
+                    tx["vout"]
+                ],
+            }
+            for tx in decoded_tx["used_utxo"]
+        ]
+        return [
+            {
+                "txid": utxo["txid"],
+                "vout": utxo["vout"],
+                "amount": utxo["details"]["value"],
+                "address": utxo["details"]["addresses"][0],
+                "label": self.getlabel(utxo["details"]["addresses"][0]),
+            }
+            for utxo in rbf_utxo
+        ]
+
+    def decode_tx(self, txid):
+        raw_tx = self.gettransaction(txid)["hex"]
+        raw_psbt = self.rpc.utxoupdatepsbt(
+            self.rpc.converttopsbt(raw_tx, True),
+            [self.recv_descriptor, self.change_descriptor],
+        )
+
+        psbt = self.rpc.decodepsbt(raw_psbt)
+        return {
+            "addresses": [
+                vout["scriptPubKey"]["addresses"][0]
+                for i, vout in enumerate(psbt["tx"]["vout"])
+                if not self.get_address_info(vout["scriptPubKey"]["addresses"][0])
+                or not self.get_address_info(
+                    vout["scriptPubKey"]["addresses"][0]
+                ).change
+            ],
+            "amounts": [
+                vout["value"]
+                for i, vout in enumerate(psbt["tx"]["vout"])
+                if not self.get_address_info(vout["scriptPubKey"]["addresses"][0])
+                or not self.get_address_info(
+                    vout["scriptPubKey"]["addresses"][0]
+                ).change
+            ],
+            "used_utxo": [
+                {"txid": vin["txid"], "vout": vin["vout"]} for vin in psbt["tx"]["vin"]
+            ],
+        }
+
+    def bumpfee(self, txid, fee_rate):
         raw_tx = self.gettransaction(txid)["hex"]
         raw_psbt = self.rpc.utxoupdatepsbt(
             self.rpc.converttopsbt(raw_tx, True),

tests/test_wallet_manager.py

@@ -126,15 +126,9 @@ def test_wallet_createpsbt(docker, request, devices_filled_data_folder, device_m
     unspents = wallet.rpc.listunspent(0)
     # Lets take 3 more or less random txs from the unspents:
     selected_coins = [
-        "{},{},{}".format(
-            unspents[5]["txid"], unspents[5]["vout"], unspents[5]["amount"]
-        ),
-        "{},{},{}".format(
-            unspents[9]["txid"], unspents[9]["vout"], unspents[9]["amount"]
-        ),
-        "{},{},{}".format(
-            unspents[12]["txid"], unspents[12]["vout"], unspents[12]["amount"]
-        ),
+        "{},{}".format(unspents[5]["txid"], unspents[5]["vout"]),
+        "{},{}".format(unspents[9]["txid"], unspents[9]["vout"]),
+        "{},{}".format(unspents[12]["txid"], unspents[12]["vout"]),
     ]
     selected_coins_amount_sum = (
         unspents[5]["amount"] + unspents[9]["amount"] + unspents[12]["amount"]