Convert jmclient to py3 style

.gitignore

@@ -6,13 +6,16 @@
 blockchain.cache
 env
 htmlcov/
+joinmarket.cfg
+joinmarket.cfg.bak
 scripts/joinmarket.cfg
 scripts/cmttools/commitments.json
 scripts/blacklist
 alicekey
 alicepubkey
 bobkey
 commitments_debug.txt
+dummyext
 deps/
 jmvenv/
 logs/

.travis.yml

@@ -16,19 +16,31 @@ matrix:
              - python-qt4 python-sip
     - os: linux
       services: docker
-      env: DOCKER_IMG_JM=xenial
+      env: DOCKER_IMG_JM=xenial-py2
     - os: linux
       services: docker
-      env: DOCKER_IMG_JM=bionic
+      env: DOCKER_IMG_JM=xenial-py3
     - os: linux
       services: docker
-      env: DOCKER_IMG_JM=stretch
+      env: DOCKER_IMG_JM=bionic-py2
     - os: linux
       services: docker
-      env: DOCKER_IMG_JM=centos7
+      env: DOCKER_IMG_JM=bionic-py3
     - os: linux
       services: docker
-      env: DOCKER_IMG_JM=fedora27
+      env: DOCKER_IMG_JM=stretch-py2
+    - os: linux
+      services: docker
+      env: DOCKER_IMG_JM=stretch-py3
+    - os: linux
+      services: docker
+      env: DOCKER_IMG_JM=centos7-py2
+    - os: linux
+      services: docker
+      env: DOCKER_IMG_JM=fedora27-py2
+    - os: linux
+      services: docker
+      env: DOCKER_IMG_JM=fedora27-py3
 before_install:
   - do_on(){ if [ "$TRAVIS_OS_NAME" = "$1" ]; then shift; $@ ; fi; }
   - on_host(){ if [ -z "$DOCKER_IMG_JM" ]; then $@ ; fi; }

install.sh

@@ -14,10 +14,18 @@ sha256_verify ()
 deps_install ()
 {
     if [[ ${install_os} == 'debian' ]]; then
-        if deb_deps_install "python-virtualenv curl python-dev python-pip build-essential automake pkg-config libtool libgmp-dev"; then
-            return 0
+        if is_python3; then
+            if deb_deps_install "python-virtualenv curl python3-dev python3-pip build-essential automake pkg-config libtool libgmp-dev"; then
+                return 0
+            else
+                return 1
+            fi
         else
-            return 1
+            if deb_deps_install "python-virtualenv curl python-dev python-pip build-essential automake pkg-config libtool libgmp-dev"; then
+                return 0
+            else
+                return 1
+            fi
         fi
     else
         echo "OS can not be determined. Trying to build."
@@ -360,6 +368,18 @@ os_is_deb ()
     ( which apt-get && which dpkg-query ) 2>/dev/null 1>&2
 }
 
+is_python3 ()
+{
+    if [[ ${python} == 'python3' ]]; then
+        return 0
+    fi
+    if eval "${python} -c 'import sys; sys.exit(0) if sys.version_info >= (3,0) else sys.exit(1)'"; then
+        return 0
+    else
+        return 1
+    fi
+}
+
 install_get_os ()
 {
     if os_is_deb; then
@@ -372,8 +392,14 @@ install_get_os ()
 qt_deps_install ()
 {
     if [[ ${install_os} == 'debian' ]]; then
-        if deb_deps_install "python-qt4 python-sip"; then
-            return 0;
+        if is_python3; then
+            if deb_deps_install "python3-pyqt4 python3-sip"; then
+                return 0;
+            fi
+        else
+            if deb_deps_install "python-qt4 python-sip"; then
+                return 0;
+            fi
         fi
     else
         return 1

jmbase/jmbase/__init__.py

@@ -3,7 +3,7 @@
 from builtins import *
 
 from .support import (get_log, chunks, debug_silence, debug_dump_object,
-                      joinmarket_alert, core_alert, get_password, _byteify,
+                      joinmarket_alert, core_alert, get_password,
                       set_logging_level)
-from commands import *
+from .commands import *
 

jmbase/jmbase/bigstring.py

@@ -1,19 +1,25 @@
 from __future__ import (absolute_import, division,
                         print_function, unicode_literals)
 from builtins import * # noqa: F401
-try:
-    from cStringIO import StringIO
-except ImportError: #pragma: no cover
-    from StringIO import StringIO #pragma: no cover
-from itertools import count
+import itertools
 from twisted.protocols import amp
 
-CHUNK_MAX = 0xffff
 
-class BigString(amp.Argument):
+def split_string(x, size):
+    return list(x[i*size:(i+1)*size] for i in range((len(x)+size-1)//size))
+
+class StringList(amp.Argument):
+    def fromBox(self, name, strings, objects, proto):
+        nk = amp._wireNameToPythonIdentifier(name)
+        objects[nk] = list(itertools.takewhile(bool, (strings.pop(b'%s.%d' % (name, i), None) for i in itertools.count())))
+
+    def toBox(self, name, strings, objects, proto):
+        for i, elem in enumerate(objects.pop(name)):
+            strings[b'%s.%d' % (name, i)] = elem
+
+class BigString(StringList):
     """
     A byte-string amp.Argument with no 65,535 length limit.
-
     Each value for a key/value pair in an AMP box may not
     exceed 65,535 bytes in length. So if we *really* want to
     send potentially larger values, this class will implicitly
@@ -22,29 +28,18 @@ class BigString(amp.Argument):
     names by prefixing this Argument's key name to a counter.
     """
     def fromBox(self, name, strings, objects, proto):
-        value = StringIO()
-        value.write(strings.get(name))
-        for counter in count(2):
-            chunk = strings.get("%s.%d" % (name, counter))
-            if chunk is None:
-                break
-            value.write(chunk)
-        objects[name] = self.buildvalue(value.getvalue())
-
-    def buildvalue(self, value):
-        return value
+        nk = amp._wireNameToPythonIdentifier(name)
+        StringList.fromBox(self, name, strings, objects, proto)
+        objects[nk] = b''.join((elem) for elem in objects[nk]).decode('utf-8')
 
     def toBox(self, name, strings, objects, proto):
-        value = StringIO(self.fromvalue(objects[name]))
-        firstChunk = value.read(CHUNK_MAX)
-        strings[name] = firstChunk
-        counter = 2
-        while True:
-            nextChunk = value.read(CHUNK_MAX)
-            if not nextChunk:
-                break
-            strings["{}.{}".format(name, counter).encode('ascii')] = nextChunk
-            counter += 1
-
-    def fromvalue(self, value):
-        return value
+        obj = self.retrieve(objects, amp._wireNameToPythonIdentifier(name), proto).encode('utf-8')
+        objects[name] = split_string(obj, amp.MAX_VALUE_LENGTH)
+        StringList.toBox(self, name, strings, objects, proto)
+
+class BigUnicode(BigString):
+    def toString(self, inObject):
+        return BigString.toString(self, inObject.encode('utf-8'))
+
+    def fromString(self, inString):
+        return BigString.fromString(self, inString).decode('utf-8')

jmbase/jmbase/commands.py

@@ -7,7 +7,7 @@
 Used for AMP asynchronous messages.
 """
 from twisted.protocols.amp import Boolean, Command, Integer, Unicode
-from .bigstring import BigString
+from .bigstring import BigUnicode
 
 class DaemonNotReady(Exception):
     pass
@@ -185,7 +185,7 @@ class JMOffers(JMCommand):
     """Return the entire contents of the
     orderbook to TAKER, as a json-ified dict.
     """
-    arguments = [(b'orderbook', BigString())]
+    arguments = [(b'orderbook', BigUnicode())]
 
 class JMFillResponse(JMCommand):
     """Returns ioauth data from MAKER if successful.

jmbase/jmbase/support.py

@@ -1,6 +1,7 @@
 from __future__ import (absolute_import, division,
                         print_function, unicode_literals)
 from builtins import * # noqa: F401
+from future.utils import iteritems
 
 import sys
 
@@ -53,13 +54,16 @@ def chunks(d, n):
     return [d[x:x + n] for x in range(0, len(d), n)]
 
 def get_password(msg): #pragma: no cover
-    return getpass(msg)
+    password = getpass(msg)
+    if not isinstance(password, bytes):
+        password = password.encode('utf-8')
+    return password
 
 def debug_dump_object(obj, skip_fields=None):
     if skip_fields is None:
         skip_fields = []
     log.debug('Class debug dump, name:' + obj.__class__.__name__)
-    for k, v in obj.__dict__.iteritems():
+    for k, v in iteritems(obj.__dict__):
         if k in skip_fields:
             continue
         if k == 'password' or k == 'given_password':
@@ -72,20 +76,3 @@ def debug_dump_object(obj, skip_fields=None):
             log.debug(pprint.pformat(v))
         else:
             log.debug(str(v))
-
-def _byteify(data, ignore_dicts = False):
-    # if this is a unicode string, return its string representation
-    if isinstance(data, unicode):
-        return data.encode('utf-8')
-    # if this is a list of values, return list of byteified values
-    if isinstance(data, list):
-        return [ _byteify(item, ignore_dicts=True) for item in data ]
-    # if this is a dictionary, return dictionary of byteified keys and values
-    # but only if we haven't already byteified it
-    if isinstance(data, dict) and not ignore_dicts:
-        return {
-            _byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True)
-            for key, value in data.iteritems()
-        }
-    # if it's anything else, return it in its original form
-    return data