Merge #536: python-bitcointx backend for jmbitcoin + bip78 and snicker.

41540ab Modify Payjoin code for BIP78 changes. (Adam Gibson)
3ed4e88 Search for correct library extension on mac os (Jules Comte)
6e6bf0a Use VERIFY_STRICTENC flag for Script verification (Adam Gibson)
55295e8 first waypoint on bip78 (Adam Gibson)
d34c53b Various fixups: (Adam Gibson)
53ef79b Updates to account for code changes in #544 (Adam Gibson)
4cf77ed Various bugfixes: (Adam Gibson)
ca0de5c Add bip78 payjoin module and client-server test: See: https://github.com/bitcoin/bips/blob/master/bip-0078.mediawiki (Adam Gibson)
ad459d2 Add human readable representations of txs and PSBTs (Adam Gibson)
03a1359 Adds libsecp256k1 installation and addresses reviews (Adam Gibson)
037a2c1 Adds full payjoin workflow test (Adam Gibson)
de3ad53 Support output of PSBT instead of broadcast in direct_send (Adam Gibson)
f060781 Add SNICKER support to wallets. (Adam Gibson)
22ed0e0 Adds psbt creation and signing support in JM wallet. (Adam Gibson)
070c5bf python-bitcointx backend for jmbitcoin. (Adam Gibson)

.gitignore

@@ -23,6 +23,7 @@ miniircd/
 miniircd.tar.gz
 nums_basepoints.txt
 schedulefortesting
+test_proposals.txt
 scripts/commitmentlist
 tmp/
 wallets/

install.sh

@@ -234,6 +234,40 @@ libffi_install ()
     popd
 }
 
+libsecp256k1_build()
+{
+    make clean
+    ./autogen.sh
+    ./configure \
+    --enable-module-recovery \
+    --disable-jni \
+    --prefix "${jm_root}" \
+    --enable-experimental \
+    --enable-module-ecdh \
+    --enable-benchmark=no
+    make
+    if ! make check; then
+        return 1
+    fi
+}
+
+libsecp256k1_install()
+{
+    secp256k1_lib_tar='0d9540b13ffcd7cd44cc361b8744b93d88aa76ba'
+    secp256k1_lib_sha="0803d2dddbf6dd702c379118f066f638bcef6b07eea959f12d31ad2f4721fbe1"
+    secp256k1_lib_url='https://github.com/bitcoin-core/secp256k1/archive'
+    if ! dep_get "${secp256k1_lib_tar}.tar.gz" "${secp256k1_lib_sha}" "${secp256k1_lib_url}"; then
+        return 1
+    fi
+    pushd "secp256k1-${secp256k1_lib_tar}"
+    if libsecp256k1_build; then
+        make install
+    else
+        return 1
+    fi
+    popd
+}
+
 libsodium_build ()
 {
     make uninstall
@@ -419,6 +453,10 @@ main ()
 #        echo "Openssl was not built. Exiting."
 #        return 1
 #    fi
+    if ! libsecp256k1_install; then
+        echo "libsecp256k1 was not built. Exiting."
+        return 1
+    fi
     if ! libffi_install; then
         echo "Libffi was not built. Exiting."
         return 1

jmbase/jmbase/__init__.py

@@ -2,7 +2,11 @@
 from .support import (get_log, chunks, debug_silence, jmprint,
                       joinmarket_alert, core_alert, get_password,
                       set_logging_level, set_logging_color,
-                      lookup_appdata_folder,
+                      lookup_appdata_folder, bintohex, bintolehex,
+                      hextobin, lehextobin, utxostr_to_utxo,
+                      utxo_to_utxostr, EXIT_ARGERROR, EXIT_FAILURE,
+                      EXIT_SUCCESS, hexbin, dictchanger, listchanger,
                       JM_WALLET_NAME_PREFIX, JM_APP_NAME)
+from .bytesprod import BytesProducer
 from .commands import *
 

jmbase/jmbase/bytesprod.py

@@ -0,0 +1,22 @@
+""" See https://twistedmatrix.com/documents/current/web/howto/client.html
+"""
+from zope.interface import implementer
+
+from twisted.internet.defer import succeed
+from twisted.web.iweb import IBodyProducer
+
+@implementer(IBodyProducer)
+class BytesProducer(object):
+    def __init__(self, body):
+        self.body = body
+        self.length = len(body)
+
+    def startProducing(self, consumer):
+        consumer.write(self.body)
+        return succeed(None)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass

jmbase/jmbase/support.py

@@ -1,8 +1,9 @@
 
 import logging, sys
+import binascii
 from getpass import getpass
 from os import path, environ
-
+from functools import wraps
 # JoinMarket version
 JM_CORE_VERSION = '0.7.0dev'
 
@@ -80,6 +81,71 @@ def emit(self, record):
 handler.setFormatter(logFormatter)
 log.addHandler(handler)
 
+# hex/binary conversion routines used by dependent packages
+def hextobin(h):
+    """Convert a hex string to bytes"""
+    return binascii.unhexlify(h.encode('utf8'))
+
+
+def bintohex(b):
+    """Convert bytes to a hex string"""
+    return binascii.hexlify(b).decode('utf8')
+
+
+def lehextobin(h):
+    """Convert a little-endian hex string to bytes
+
+    Lets you write uint256's and uint160's the way the Satoshi codebase shows
+    them.
+    """
+    return binascii.unhexlify(h.encode('utf8'))[::-1]
+
+
+def bintolehex(b):
+    """Convert bytes to a little-endian hex string
+
+    Lets you show uint256's and uint160's the way the Satoshi codebase shows
+    them.
+    """
+    return binascii.hexlify(b[::-1]).decode('utf8')
+
+def utxostr_to_utxo(x):
+    if not isinstance(x, str):
+        return (False, "not a string")
+    y = x.split(":")
+    if len(y) != 2:
+        return (False,
+                "string is not two items separated by :")
+    try:
+        n = int(y[1])
+    except:
+        return (False, "utxo index was not an integer.")
+    if n < 0:
+        return (False, "utxo index must not be negative.")
+    if len(y[0]) != 64:
+        return (False, "txid is not 64 hex characters.")
+    try:
+        txid = binascii.unhexlify(y[0])
+    except:
+        return (False, "txid is not hex.")
+    return (True, (txid, n))
+
+def utxo_to_utxostr(u):
+    if not isinstance(u, tuple):
+        return (False, "utxo is not a tuple.")
+    if not len(u) == 2:
+        return (False, "utxo should have two elements.")
+    if not isinstance(u[0], bytes):
+        return (False, "txid should be bytes.")
+    if not isinstance(u[1], int):
+        return (False, "index should be int.")
+    if u[1] < 0:
+        return (False, "index must be a positive integer.")
+    if not len(u[0]) == 32:
+        return (False, "txid must be 32 bytes.")
+    txid = binascii.hexlify(u[0]).decode("ascii")
+    return (True, txid + ":" + str(u[1]))
+
 def jmprint(msg, level="info"):
     """ Provides the ability to print messages
     with consistent formatting, outside the logging system
@@ -150,3 +216,67 @@ def lookup_appdata_folder(appname):
 def print_jm_version(option, opt_str, value, parser):
     print("JoinMarket " + JM_CORE_VERSION)
     sys.exit(EXIT_SUCCESS)
+
+# helper functions for conversions of format between over-the-wire JM
+# and internal. See details in hexbin() docstring.
+
+def _convert(x):
+    good, utxo = utxostr_to_utxo(x)
+    if good:
+        return utxo
+    else:
+        try:
+            b = hextobin(x)
+            return b
+        except:
+            return x
+
+def listchanger(l):
+    rlist = []
+    for x in l:
+        if isinstance(x, list):
+            rlist.append(listchanger(x))
+        elif isinstance(x, dict):
+            rlist.append(dictchanger(x))
+        else:
+            rlist.append(_convert(x))
+    return rlist
+
+def dictchanger(d):
+    rdict = {}
+    for k, v in d.items():
+        if isinstance(v, dict):
+            rdict[_convert(k)] = dictchanger(v)
+        elif isinstance(v, list):
+            rdict[_convert(k)] = listchanger(v)
+        else:
+            rdict[_convert(k)] = _convert(v)
+    return rdict
+
+def hexbin(func):
+    """ Decorator for functions of taker and maker receiving over
+    the wire AMP arguments that may be in hex or hextxid:n format
+    and converting all to binary.
+    Functions to which this decorator applies should have all arguments
+    be one of:
+    - hex string (keys), converted here to binary
+    - lists of keys or txid:n strings (converted here to binary, or
+      (txidbytes, n))
+    - lists of lists or dicts, to which these rules apply recursively.
+    - any other string (unchanged)
+    - dicts with keys as per above; values are altered recursively according
+      to the rules above.
+    """
+    @wraps(func)
+    def func_wrapper(inst, *args, **kwargs):
+        newargs = []
+        for arg in args:
+            if isinstance(arg, (list, tuple)):
+                newargs.append(listchanger(arg))
+            elif isinstance(arg, dict):
+                newargs.append(dictchanger(arg))
+            else:
+                newargs.append(_convert(arg))
+        return func(inst, *newargs, **kwargs)
+
+    return func_wrapper

jmbase/setup.py

@@ -9,7 +9,7 @@
       author_email='',
       license='GPL',
       packages=['jmbase'],
-      install_requires=['future', 'twisted==19.7.0', 'service-identity',
+      install_requires=['twisted==19.7.0', 'service-identity',
                         'chromalog==1.0.5'],
-      python_requires='>=3.3',
+      python_requires='>=3.6',
       zip_safe=False)

jmbitcoin/jmbitcoin/__init__.py

@@ -1,9 +1,35 @@
 import coincurve as secp256k1
+
+# If user has compiled and installed libsecp256k1 via
+# JM installation script install.sh, use that;
+# if not, it is assumed to be present at the system level
+# See: https://github.com/Simplexum/python-bitcointx/commit/79333106eeb55841df2935781646369b186d99f7#diff-1ea6586127522e62d109ec5893a18850R301-R310
+import os, sys
+if sys.platform == "darwin":
+    secp_name = "libsecp256k1.dylib"
+else:
+    secp_name = "libsecp256k1.so"
+expected_secp_location = os.path.join(sys.prefix, "lib", secp_name)
+if os.path.exists(expected_secp_location):
+    import bitcointx
+    bitcointx.set_custom_secp256k1_path(expected_secp_location)
+
 from jmbitcoin.secp256k1_main import *
 from jmbitcoin.secp256k1_transaction import *
 from jmbitcoin.secp256k1_deterministic import *
-from jmbitcoin.btscript import *
-from jmbitcoin.bech32 import *
+from jmbitcoin.snicker import *
 from jmbitcoin.amount import *
 from jmbitcoin.bip21 import *
+from bitcointx import select_chain_params
+from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn,
+                            CTxInWitness, CTxWitness, CTransaction,
+                            CMutableTransaction, Hash160,
+                            coins_to_satoshi, satoshi_to_coins)
+from bitcointx.core.key import KeyStore
+from bitcointx.wallet import (P2SHCoinAddress, P2SHCoinAddressError,
+                              P2WPKHCoinAddress, P2WPKHCoinAddressError)
+from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL,
+                                   SIGVERSION_WITNESS_V0, CScriptWitness)
+from bitcointx.core.psbt import (PartiallySignedTransaction, PSBT_Input,
+                                 PSBT_Output)