feat: add atomic swap refund transaction handling (#3573)

Description
---
This Pr adds the following things:
Atomic swap refund transactions
Proper handling of the script context
Transaction spend priority
SQL query for filtering locked outputs
Fix broken get_balance tests


Motivation and Context
---
One of the features that are required for atomic swaps is to be able to process refunds. This PR now allows atomic swap refunds to be processed after the time lock of the HTLC has passed. This can happen in a manual way where the user manually asks the console-wallet via the CLI, or if the time lock has passed the wallet will automatically spend this UTXO first. 

One of the missing features of TariScript is the `scriptContext`, which allows scripts to validate the blockchain state such as block height that is required in this case. This is now piped in anywhere where the script is validated if possible. If the state cannot be gathered, the state is filled in with default values which is how it operated up to now. 

This adds in a transaction spend priority that can allow UTXO to be given a priority to be spent. This is the case for high-risk UTXO such as HTLC which more than one party can claim. Manual claim for these transactions is still advised as they can be claimed as soon as possible, but this adds a second automated state where if you want to spend some UTXO at least pick the ones that you need to spend more urgently than the other ones. 

The current `select_utxos` function uses a rust function to filter the maturity of the UTXOs to only select UTXOs it can spend. This PR completely swaps out the selection to be based completely on SQL queries which should be much faster. 

All current get_balance tests are broken in where the available and time-locked balance is asserted to be the same. This is not due to the function being broken but rather the test and the testing code. What happened is that tests use a tip_height of `u64::MAX` but this is too large as the database uses `i64::MAX`. The problem comes when `u64::MAX` is cast as `i64` as this is equal to `-1` and when the query `FROM outputs WHERE status = ? AND maturity > ? AND script_lock_height > ?` is run, it select everything as the tip is now `-1`. This ensures that the testing tip is selected as `i64::MAX` to let the queries run correctly.

How Has This Been Tested?
---
Added new tests.

applications/tari_app_grpc/proto/types.proto

@@ -288,6 +288,8 @@ message UnblindedOutput {
     bytes sender_offset_public_key = 8;
     // UTXO signature with the script offset private key, k_O
     ComSignature metadata_signature = 9;
+    // The minimum height the script allows this output to be spent
+    uint64 script_lock_height = 10;
 }
 
 // ----------------------------- Network Types ----------------------------- //

applications/tari_app_grpc/proto/wallet.proto

@@ -54,12 +54,14 @@ service Wallet {
     rpc ListConnectedPeers(Empty) returns (ListConnectedPeersResponse);
     // Cancel pending transaction
     rpc CancelTransaction (CancelTransactionRequest) returns (CancelTransactionResponse);
-    // Will triggger a complete revalidation of all wallet outputs.
+    // Will trigger a complete revalidation of all wallet outputs.
     rpc RevalidateAllTransactions (RevalidateRequest) returns (RevalidateResponse);
     // This will send a XTR SHA Atomic swap transaction
     rpc SendShaAtomicSwapTransaction(SendShaAtomicSwapRequest) returns (SendShaAtomicSwapResponse);
     // This will claim a XTR SHA Atomic swap transaction
     rpc ClaimShaAtomicSwapTransaction(ClaimShaAtomicSwapRequest) returns (ClaimShaAtomicSwapResponse);
+    // This will claim a HTLC refund transaction
+    rpc ClaimHtlcRefundTransaction(ClaimHtlcRefundRequest) returns (ClaimHtlcRefundResponse);
 }
 
 message GetVersionRequest { }
@@ -125,6 +127,15 @@ message ClaimShaAtomicSwapResponse {
     TransferResult results = 1;
 }
 
+message ClaimHtlcRefundRequest{
+    string output_hash = 1;
+    uint64 fee_per_gram = 2;
+}
+
+message ClaimHtlcRefundResponse {
+    TransferResult results = 1;
+}
+
 message GetTransactionInfoRequest {
     repeated uint64 transaction_ids = 1;
 }

applications/tari_app_grpc/src/conversions/unblinded_output.rs

@@ -49,6 +49,7 @@ impl From<UnblindedOutput> for grpc::UnblindedOutput {
                 signature_u: Vec::from(output.metadata_signature.u().as_bytes()),
                 signature_v: Vec::from(output.metadata_signature.v().as_bytes()),
             }),
+            script_lock_height: output.script_lock_height,
         }
     }
 }
@@ -91,6 +92,7 @@ impl TryFrom<grpc::UnblindedOutput> for UnblindedOutput {
             script_private_key,
             sender_offset_public_key,
             metadata_signature,
+            script_lock_height: output.script_lock_height,
         })
     }
 }

applications/tari_base_node/src/builder.rs

@@ -246,6 +246,7 @@ async fn build_node_context(
         Box::new(TxInternalConsistencyValidator::new(
             factories.clone(),
             config.base_node_bypass_range_proof_verification,
+            blockchain_db.clone(),
         )),
         Box::new(TxInputAndMaturityValidator::new(blockchain_db.clone())),
         Box::new(TxConsensusValidator::new(blockchain_db.clone())),

applications/tari_console_wallet/src/automation/command_parser.rs

@@ -59,6 +59,7 @@ impl Display for ParsedCommand {
             ClearCustomBaseNode => "clear-custom-base-node",
             InitShaAtomicSwap => "init-sha-atomic-swap",
             FinaliseShaAtomicSwap => "finalise-sha-atomic-swap",
+            ClaimShaAtomicSwapRefund => "claim-sha-atomic-swap-refund",
         };
 
         let args = self
@@ -130,6 +131,7 @@ pub fn parse_command(command: &str) -> Result<ParsedCommand, ParseError> {
         ClearCustomBaseNode => Vec::new(),
         InitShaAtomicSwap => parse_init_sha_atomic_swap(args)?,
         FinaliseShaAtomicSwap => parse_finalise_sha_atomic_swap(args)?,
+        ClaimShaAtomicSwapRefund => parse_claim_htlc_refund_refund(args)?,
     };
 
     Ok(ParsedCommand { command, args })
@@ -219,6 +221,18 @@ fn parse_finalise_sha_atomic_swap(mut args: SplitWhitespace) -> Result<Vec<Parse
     Ok(parsed_args)
 }
 
+fn parse_claim_htlc_refund_refund(mut args: SplitWhitespace) -> Result<Vec<ParsedArgument>, ParseError> {
+    let mut parsed_args = Vec::new();
+    // hash
+    let hash = args
+        .next()
+        .ok_or_else(|| ParseError::Empty("Output hash".to_string()))?;
+    let hash = parse_hash(hash).ok_or(ParseError::Hash)?;
+    parsed_args.push(ParsedArgument::Hash(hash));
+
+    Ok(parsed_args)
+}
+
 fn parse_make_it_rain(mut args: SplitWhitespace) -> Result<Vec<ParsedArgument>, ParseError> {
     let mut parsed_args = Vec::new();
 

applications/tari_console_wallet/src/automation/commands.rs

@@ -88,6 +88,7 @@ pub enum WalletCommand {
     ClearCustomBaseNode,
     InitShaAtomicSwap,
     FinaliseShaAtomicSwap,
+    ClaimShaAtomicSwapRefund,
 }
 
 #[derive(Debug, EnumString, PartialEq, Clone)]
@@ -205,6 +206,27 @@ pub async fn finalise_sha_atomic_swap(
     Ok(tx_id)
 }
 
+/// claims a HTLC refund transaction
+pub async fn claim_htlc_refund(
+    mut output_service: OutputManagerHandle,
+    mut transaction_service: TransactionServiceHandle,
+    args: Vec<ParsedArgument>,
+) -> Result<TxId, CommandError> {
+    use ParsedArgument::*;
+    let output = match args[0].clone() {
+        Hash(output) => Ok(output),
+        _ => Err(CommandError::Argument),
+    }?;
+
+    let (tx_id, fee, amount, tx) = output_service
+        .create_htlc_refund_transaction(output, MicroTari(25))
+        .await?;
+    transaction_service
+        .submit_transaction(tx_id, tx, fee, amount, "Claimed HTLC refund".into())
+        .await?;
+    Ok(tx_id)
+}
+
 /// Send a one-sided transaction to a recipient
 pub async fn send_one_sided(
     mut wallet_transaction_service: TransactionServiceHandle,
@@ -759,6 +781,11 @@ pub async fn command_runner(
                 debug!(target: LOG_TARGET, "claiming tari HTLC tx_id {}", tx_id);
                 tx_ids.push(tx_id);
             },
+            ClaimShaAtomicSwapRefund => {
+                let tx_id = claim_htlc_refund(output_service.clone(), transaction_service.clone(), parsed.args).await?;
+                debug!(target: LOG_TARGET, "claiming tari HTLC tx_id {}", tx_id);
+                tx_ids.push(tx_id);
+            },
         }
     }
 

applications/tari_console_wallet/src/grpc/wallet_grpc_server.rs

@@ -7,6 +7,8 @@ use tari_app_grpc::{
     tari_rpc::{
         payment_recipient::PaymentType,
         wallet_server,
+        ClaimHtlcRefundRequest,
+        ClaimHtlcRefundResponse,
         ClaimShaAtomicSwapRequest,
         ClaimShaAtomicSwapResponse,
         CoinSplitRequest,
@@ -189,7 +191,7 @@ impl wallet_server::Wallet for WalletGrpcServer {
                     "Transaction broadcast: {}, preimage_hex: {}, hash {}",
                     tx_id,
                     pre_image.to_hex(),
-                    output.to_string()
+                    output.hash().to_hex()
                 );
                 SendShaAtomicSwapResponse {
                     transaction_id: tx_id,
@@ -226,7 +228,7 @@ impl wallet_server::Wallet for WalletGrpcServer {
             .map_err(|_| Status::internal("pre_image is malformed".to_string()))?;
         let output = BlockHash::from_hex(&message.output)
             .map_err(|_| Status::internal("Output hash is malformed".to_string()))?;
-
+        debug!(target: LOG_TARGET, "Trying to claim HTLC with hash {}", output.to_hex());
         let mut transaction_service = self.get_transaction_service();
         let mut output_manager_service = self.get_output_manager_service();
         let response = match output_manager_service
@@ -274,6 +276,56 @@ impl wallet_server::Wallet for WalletGrpcServer {
         }))
     }
 
+    async fn claim_htlc_refund_transaction(
+        &self,
+        request: Request<ClaimHtlcRefundRequest>,
+    ) -> Result<Response<ClaimHtlcRefundResponse>, Status> {
+        let message = request.into_inner();
+        let output = BlockHash::from_hex(&message.output_hash)
+            .map_err(|_| Status::internal("Output hash is malformed".to_string()))?;
+
+        let mut transaction_service = self.get_transaction_service();
+        let mut output_manager_service = self.get_output_manager_service();
+        debug!(target: LOG_TARGET, "Trying to claim HTLC with hash {}", output.to_hex());
+        let response = match output_manager_service
+            .create_htlc_refund_transaction(output, message.fee_per_gram.into())
+            .await
+        {
+            Ok((tx_id, fee, amount, tx)) => {
+                match transaction_service
+                    .submit_transaction(tx_id, tx, fee, amount, "Creating HTLC refund transaction".to_string())
+                    .await
+                {
+                    Ok(()) => TransferResult {
+                        address: Default::default(),
+                        transaction_id: tx_id,
+                        is_success: true,
+                        failure_message: Default::default(),
+                    },
+                    Err(e) => TransferResult {
+                        address: Default::default(),
+                        transaction_id: Default::default(),
+                        is_success: false,
+                        failure_message: e.to_string(),
+                    },
+                }
+            },
+            Err(e) => {
+                warn!(target: LOG_TARGET, "Failed to claim HTLC refund transaction: {}", e);
+                TransferResult {
+                    address: Default::default(),
+                    transaction_id: Default::default(),
+                    is_success: false,
+                    failure_message: e.to_string(),
+                }
+            },
+        };
+
+        Ok(Response::new(ClaimHtlcRefundResponse {
+            results: Some(response),
+        }))
+    }
+
     async fn transfer(&self, request: Request<TransferRequest>) -> Result<Response<TransferResponse>, Status> {
         let message = request.into_inner();
         let recipients = message

base_layer/core/src/mempool/unconfirmed_pool/unconfirmed_pool.rs

@@ -596,8 +596,12 @@ mod test {
             .unwrap();
 
         let factories = CryptoFactories::default();
-        let mut stx_protocol = stx_builder.build::<HashDigest>(&factories).unwrap();
-        stx_protocol.finalize(KernelFeatures::empty(), &factories).unwrap();
+        let mut stx_protocol = stx_builder
+            .build::<HashDigest>(&factories, None, Some(u64::MAX))
+            .unwrap();
+        stx_protocol
+            .finalize(KernelFeatures::empty(), &factories, None, Some(u64::MAX))
+            .unwrap();
 
         let tx3 = stx_protocol.get_transaction().unwrap().clone();
 

base_layer/core/src/transactions/aggregated_body.rs

@@ -41,12 +41,14 @@ use log::*;
 use serde::{Deserialize, Serialize};
 use std::{
     cmp::max,
+    convert::TryInto,
     fmt::{Display, Error, Formatter},
 };
 use tari_common_types::types::{
     BlindingFactor,
     Commitment,
     CommitmentFactory,
+    HashOutput,
     PrivateKey,
     PublicKey,
     RangeProofService,
@@ -55,6 +57,7 @@ use tari_crypto::{
     commitment::HomomorphicCommitmentFactory,
     keys::PublicKey as PublicKeyTrait,
     ristretto::pedersen::PedersenCommitment,
+    script::ScriptContext,
     tari_utilities::hex::Hex,
 };
 
@@ -342,13 +345,16 @@ impl AggregateBody {
     /// This function does NOT check that inputs come from the UTXO set
     /// The reward is the total amount of Tari rewarded for this block (block reward + total fees), this should be 0
     /// for a transaction
+    #[allow(clippy::too_many_arguments)]
     pub fn validate_internal_consistency(
         &self,
         tx_offset: &BlindingFactor,
         script_offset: &BlindingFactor,
         bypass_range_proof_verification: bool,
         total_reward: MicroTari,
         factories: &CryptoFactories,
+        prev_header: Option<HashOutput>,
+        height: Option<u64>,
     ) -> Result<(), TransactionError> {
         self.verify_kernel_signatures()?;
 
@@ -361,7 +367,7 @@ impl AggregateBody {
         self.verify_metadata_signatures()?;
 
         let script_offset_g = PublicKey::from_secret_key(script_offset);
-        self.validate_script_offset(script_offset_g, &factories.commitment)
+        self.validate_script_offset(script_offset_g, &factories.commitment, prev_header, height)
     }
 
     pub fn dissolve(self) -> (Vec<TransactionInput>, Vec<TransactionOutput>, Vec<TransactionKernel>) {
@@ -425,12 +431,17 @@ impl AggregateBody {
         &self,
         script_offset: PublicKey,
         factory: &CommitmentFactory,
+        prev_header: Option<HashOutput>,
+        height: Option<u64>,
     ) -> Result<(), TransactionError> {
         trace!(target: LOG_TARGET, "Checking script offset");
         // lets count up the input script public keys
         let mut input_keys = PublicKey::default();
+        let prev_hash: [u8; 32] = prev_header.unwrap_or_default().as_slice().try_into().unwrap_or([0; 32]);
+        let height = height.unwrap_or_default();
         for input in &self.inputs {
-            input_keys = input_keys + input.run_and_verify_script(factory)?;
+            let context = ScriptContext::new(height, &prev_hash, &input.commitment);
+            input_keys = input_keys + input.run_and_verify_script(factory, Some(context))?;
         }
 
         // Now lets gather the output public keys and hashes.

base_layer/core/src/transactions/coinbase_builder.rs

@@ -208,6 +208,7 @@ impl CoinbaseBuilder {
             script_private_key,
             sender_offset_public_key,
             metadata_sig,
+            0,
         );
         let output = if let Some(rewind_data) = self.rewind_data.as_ref() {
             unblinded_output
@@ -235,7 +236,7 @@ impl CoinbaseBuilder {
             .with_reward(total_reward)
             .with_kernel(kernel);
         let tx = builder
-            .build(&self.factories)
+            .build(&self.factories, None, Some(height))
             .map_err(|e| CoinbaseBuildError::BuildError(e.to_string()))?;
         Ok((tx, unblinded_output))
     }
@@ -525,7 +526,9 @@ mod test {
                 &PrivateKey::default(),
                 false,
                 block_reward,
-                &factories
+                &factories,
+                None,
+                Some(u64::MAX)
             ),
             Ok(())
         );

base_layer/core/src/transactions/test_helpers.rs

@@ -155,6 +155,7 @@ impl TestParams {
             self.script_private_key.clone(),
             self.sender_offset_public_key.clone(),
             metadata_signature,
+            0,
         )
     }
 
@@ -444,8 +445,10 @@ pub fn create_transaction_with(
         stx_builder.with_output(utxo, script_offset_pvt_key).unwrap();
     });
 
-    let mut stx_protocol = stx_builder.build::<Blake256>(&factories).unwrap();
-    stx_protocol.finalize(KernelFeatures::empty(), &factories).unwrap();
+    let mut stx_protocol = stx_builder.build::<Blake256>(&factories, None, Some(u64::MAX)).unwrap();
+    stx_protocol
+        .finalize(KernelFeatures::empty(), &factories, None, Some(u64::MAX))
+        .unwrap();
     stx_protocol.take_transaction().unwrap()
 }
 
@@ -513,7 +516,7 @@ pub fn spend_utxos(schema: TransactionSchema) -> (Transaction, Vec<UnblindedOutp
             .unwrap();
     }
 
-    let mut stx_protocol = stx_builder.build::<Blake256>(&factories).unwrap();
+    let mut stx_protocol = stx_builder.build::<Blake256>(&factories, None, Some(u64::MAX)).unwrap();
     let change = stx_protocol.get_change_amount().unwrap();
     // The change output is assigned its own random script offset private key
     let change_sender_offset_public_key = stx_protocol.get_change_sender_offset_public_key().unwrap().unwrap();
@@ -539,9 +542,12 @@ pub fn spend_utxos(schema: TransactionSchema) -> (Transaction, Vec<UnblindedOutp
         test_params_change_and_txn.script_private_key.clone(),
         change_sender_offset_public_key,
         metadata_sig,
+        0,
     );
     outputs.push(change_output);
-    stx_protocol.finalize(KernelFeatures::empty(), &factories).unwrap();
+    stx_protocol
+        .finalize(KernelFeatures::empty(), &factories, None, Some(u64::MAX))
+        .unwrap();
     let txn = stx_protocol.get_transaction().unwrap().clone();
     (txn, outputs, test_params_change_and_txn)
 }