Merge #3765

3765: Support coin-selection of Byron inputs in `balanceTransaction` r=Anviking a=Anviking

- [x] Extend `estimateTxFee` and `estimateNumberOfWitnesses` to also be able to deal with Bootstrap witnesses
- [x] Make `produces balanced transactions or fails` property test both `Shelley` and `Byron` wallets

### Comments

Needed for #3746 

### Issue Number

ADP-2268 / ADP-2613


Co-authored-by: Johannes Lund <johannes.lund@iohk.io>

lib/wallet/src/Cardano/Wallet/Shelley/Transaction.hs

@@ -1131,14 +1131,22 @@ estimateKeyWitnessCount utxo txbody@(Cardano.TxBody txbodycontent) =
             fromIntegral
             $ sumVia estimateMaxWitnessRequiredPerInput
             $ mapMaybe toTimelockScript scripts
-    in
-    numberOfShelleyWitnesses $ fromIntegral $
-        length vkInsUnique +
-        length txExtraKeyWits' +
-        length txWithdrawals' +
-        txUpdateProposal' +
-        txCerts +
-        scriptVkWitsUpperBound
+        nonInputWits = numberOfShelleyWitnesses $ fromIntegral $
+            length txExtraKeyWits' +
+            length txWithdrawals' +
+            txUpdateProposal' +
+            txCerts +
+            scriptVkWitsUpperBound
+        inputWits = KeyWitnessCount
+            { nKeyWits = fromIntegral
+                . length
+                $ filter (not . hasBootstrapAddr utxo) vkInsUnique
+            , nBootstrapWits = fromIntegral
+                . length
+                $ filter (hasBootstrapAddr utxo) vkInsUnique
+            }
+        in
+            nonInputWits <> inputWits
   where
     scripts = case txbody of
         Cardano.ShelleyTxBody _ _ shelleyBodyScripts _ _ _ -> shelleyBodyScripts
@@ -1187,6 +1195,21 @@ estimateKeyWitnessCount utxo txbody@(Cardano.TxBody txbodycontent) =
                 , "Caller is expected to ensure this does not happen."
                 ]
 
+    hasBootstrapAddr
+        :: Cardano.UTxO era
+        -> Cardano.TxIn
+        -> Bool
+    hasBootstrapAddr (Cardano.UTxO u) inp = case Map.lookup inp u of
+        Just (Cardano.TxOut addrInEra _ _ _) ->
+            case addrInEra of
+                Cardano.AddressInEra Cardano.ByronAddressInAnyEra _ -> True
+                _ -> False
+        Nothing ->
+            error $ unwords
+                [ "estimateMaxWitnessRequiredPerInput: input not in utxo."
+                , "Caller is expected to ensure this does not happen."
+                ]
+
 maxScriptExecutionCost
     :: ProtocolParameters
      -- ^ Current protocol parameters