THRIFT-1857 Python 3 Support

Client: Python
Patch: Thomas Bartelmess, Eevee (Alex Munroe), helgridly, Christian Verkerk, Jeroen Vlek, Nobuaki Sukegawa

This closes #213 and closes #680

build/docker/ubuntu/Dockerfile

@@ -37,7 +37,7 @@ RUN apt-get install -y ant openjdk-7-jdk maven && \
 
 # Python dependencies
 RUN apt-get install -y python-all python-all-dev python-all-dbg python-setuptools python-support \
-    python-twisted python-zope.interface
+    python-twisted python-zope.interface python-six
 
 # Ruby dependencies
 RUN apt-get install -y ruby ruby-dev && \

build/travis/installDependencies.sh

@@ -27,7 +27,7 @@ sudo apt-get install -qq ant openjdk-7-jdk
 sudo update-java-alternatives -s java-1.7.0-openjdk-amd64
 
 # Python dependencies
-sudo apt-get install -qq python-all python-all-dev python-all-dbg python-setuptools python-support python-twisted
+sudo apt-get install -qq python-all python-all-dev python-all-dbg python-setuptools python-support python-twisted python-six
 
 # Ruby dependencies
 sudo apt-get install -qq ruby ruby-dev

compiler/cpp/src/generate/t_py_generator.cc

@@ -349,11 +349,18 @@ void t_py_generator::init_generator() {
   f_init.close();
 
   // Print header
-  f_types_ << py_autogen_comment() << endl << py_imports() << endl << render_includes() << endl
-           << render_fastbinary_includes() << endl << endl;
-
-  f_consts_ << py_autogen_comment() << endl << py_imports() << endl << "from ttypes import *"
-            << endl << endl;
+  f_types_ <<
+    py_autogen_comment() << endl <<
+    py_imports() << endl <<
+    render_includes() << endl <<
+    render_fastbinary_includes() <<
+    endl << endl;
+
+  f_consts_ <<
+    py_autogen_comment() << endl <<
+    py_imports() << endl <<
+    "from .ttypes import *" << endl <<
+    endl;
 }
 
 /**
@@ -759,10 +766,12 @@ void t_py_generator::generate_py_struct_definition(ofstream& out,
   if (!gen_slots_) {
     // Printing utilities so that on the command line thrift
     // structs look pretty like dictionaries
-    out << indent() << "def __repr__(self):" << endl << indent() << "  L = ['%s=%r' % (key, value)"
-        << endl << indent() << "    for key, value in self.__dict__.iteritems()]" << endl
-        << indent() << "  return '%s(%s)' % (self.__class__.__name__, ', '.join(L))" << endl
-        << endl;
+    out <<
+      indent() << "def __repr__(self):" << endl <<
+      indent() << "  L = ['%s=%r' % (key, value)" << endl <<
+      indent() << "    for key, value in self.__dict__.items()]" << endl <<
+      indent() << "  return '%s(%s)' % (self.__class__.__name__, ', '.join(L))" << endl <<
+      endl;
 
     // Equality and inequality methods that compare by value
     out << indent() << "def __eq__(self, other):" << endl;
@@ -961,7 +970,7 @@ void t_py_generator::generate_service(t_service* tservice) {
   }
 
   f_service_ << "import logging" << endl
-             << "from ttypes import *" << endl
+             << "from .ttypes import *" << endl
              << "from thrift.Thrift import TProcessor" << endl
              << render_fastbinary_includes() << endl;
 
@@ -1141,26 +1150,32 @@ void t_py_generator::generate_service_client(t_service* tservice) {
   }
 
   if (gen_tornado_ && extends.empty()) {
-    f_service_ << indent() << "@gen.engine" << endl << indent()
-               << "def _start_receiving(self):" << endl << indent() << "  while True:" << endl
-               << indent() << "    try:" << endl << indent()
-               << "      frame = yield self._transport.readFrame()" << endl << indent()
-               << "    except TTransport.TTransportException as e:" << endl << indent()
-               << "      for future in self._reqs.itervalues():" << endl << indent()
-               << "        future.set_exception(e)" << endl << indent() << "      self._reqs = {}"
-               << endl << indent() << "      return" << endl << indent()
-               << "    tr = TTransport.TMemoryBuffer(frame)" << endl << indent()
-               << "    iprot = self._iprot_factory.getProtocol(tr)" << endl << indent()
-               << "    (fname, mtype, rseqid) = iprot.readMessageBegin()" << endl << indent()
-               << "    future = self._reqs.pop(rseqid, None)" << endl << indent()
-               << "    if not future:" << endl << indent()
-               << "      # future has already been discarded" << endl << indent()
-               << "      continue" << endl << indent()
-               << "    method = getattr(self, 'recv_' + fname)" << endl << indent()
-               << "    try:" << endl << indent() << "      result = method(iprot, mtype, rseqid)"
-               << endl << indent() << "    except Exception as e:" << endl << indent()
-               << "      future.set_exception(e)" << endl << indent() << "    else:" << endl
-               << indent() << "      future.set_result(result)" << endl << endl;
+    f_service_ <<
+      indent() << "@gen.engine" << endl <<
+      indent() << "def _start_receiving(self):" << endl <<
+      indent() << "  while True:" << endl <<
+      indent() << "    try:" << endl <<
+      indent() << "      frame = yield self._transport.readFrame()" << endl <<
+      indent() << "    except TTransport.TTransportException as e:" << endl <<
+      indent() << "      for future in self._reqs.values():" << endl <<
+      indent() << "        future.set_exception(e)" << endl <<
+      indent() << "      self._reqs = {}" << endl <<
+      indent() << "      return" << endl <<
+      indent() << "    tr = TTransport.TMemoryBuffer(frame)" << endl <<
+      indent() << "    iprot = self._iprot_factory.getProtocol(tr)" << endl <<
+      indent() << "    (fname, mtype, rseqid) = iprot.readMessageBegin()" << endl <<
+      indent() << "    method = getattr(self, 'recv_' + fname)" << endl <<
+      indent() << "    future = self._reqs.pop(rseqid, None)" << endl <<
+      indent() << "    if not future:" << endl <<
+      indent() << "      # future has already been discarded" << endl <<
+      indent() << "      continue" << endl <<
+      indent() << "    try:" << endl <<
+      indent() << "      result = method(iprot, mtype, rseqid)" << endl <<
+      indent() << "    except Exception as e:" << endl <<
+      indent() << "      future.set_exception(e)" << endl <<
+      indent() << "    else:" << endl <<
+      indent() << "      future.set_result(result)" << endl <<
+      endl;
   }
 
   // Generate client method implementations
@@ -1409,21 +1424,33 @@ void t_py_generator::generate_service_remote(t_service* tservice) {
   ofstream f_remote;
   f_remote.open(f_remote_name.c_str());
 
-  f_remote << "#!/usr/bin/env python" << endl << py_autogen_comment() << endl << "import sys"
-           << endl << "import pprint" << endl << "from urlparse import urlparse" << endl
-           << "from thrift.transport import TTransport" << endl
-           << "from thrift.transport import TSocket" << endl
-           << "from thrift.transport import TSSLSocket" << endl
-           << "from thrift.transport import THttpClient" << endl
-           << "from thrift.protocol import TBinaryProtocol" << endl << endl;
-
-  f_remote << "from " << module_ << " import " << service_name_ << endl << "from " << module_
-           << ".ttypes import *" << endl << endl;
-
-  f_remote << "if len(sys.argv) <= 1 or sys.argv[1] == '--help':" << endl << "  print('')" << endl
-           << "  print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] "
-              "function [arg1 [arg2...]]')" << endl << "  print('')" << endl
-           << "  print('Functions:')" << endl;
+  f_remote <<
+    "#!/usr/bin/env python" << endl <<
+    py_autogen_comment() << endl <<
+    "import sys" << endl <<
+    "import pprint" << endl <<
+    "if sys.version_info[0] == 3:" << endl <<
+    "  from urllib.parse import urlparse" << endl <<
+    "else:" << endl <<
+    "  from urlparse import urlparse" << endl <<
+    "from thrift.transport import TTransport" << endl <<
+    "from thrift.transport import TSocket" << endl <<
+    "from thrift.transport import TSSLSocket" << endl <<
+    "from thrift.transport import THttpClient" << endl <<
+    "from thrift.protocol import TBinaryProtocol" << endl <<
+    endl;
+
+  f_remote <<
+    "from " << module_ << " import " << service_name_ << endl <<
+    "from " << module_ << ".ttypes import *" << endl <<
+    endl;
+
+  f_remote <<
+    "if len(sys.argv) <= 1 or sys.argv[1] == '--help':" << endl <<
+    "  print('')" << endl <<
+    "  print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] function [arg1 [arg2...]]')" << endl <<
+    "  print('')" << endl <<
+    "  print('Functions:')" << endl;
   for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) {
     f_remote << "  print('  " << (*f_iter)->get_returntype()->get_name() << " "
              << (*f_iter)->get_name() << "(";
@@ -1720,8 +1747,8 @@ void t_py_generator::generate_process_function(t_service* tservice, t_function*
       // Kinda absurd
       f_service_ << indent() << "  error.raiseException()" << endl;
       for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) {
-        f_service_ << indent() << "except " << type_name((*x_iter)->get_type()) << " as "
-                   << (*x_iter)->get_name() << ":" << endl;
+        f_service_ <<
+          indent() << "except " << type_name((*x_iter)->get_type()) << " as " << (*x_iter)->get_name() << ":" << endl;
         if (!tfunction->is_oneway()) {
           indent_up();
           f_service_ << indent() << "result." << (*x_iter)->get_name() << " = "
@@ -1854,11 +1881,15 @@ void t_py_generator::generate_process_function(t_service* tservice, t_function*
       for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) {
         f_service_ << indent() << "except " << type_name((*x_iter)->get_type()) << " as "
                    << (*x_iter)->get_name() << ":" << endl;
-        indent_up();
-        f_service_ << indent() << "msg_type = TMessageType.REPLY" << endl
-                   << indent() << "result." << (*x_iter)->get_name() << " = "
-                   << (*x_iter)->get_name() << endl;
-        indent_down();
+        if (!tfunction->is_oneway()) {
+          indent_up();
+          f_service_ << indent() << "msg_type = TMessageType.REPLY" << endl;
+          f_service_ << indent() << "result." << (*x_iter)->get_name() << " = "
+                     << (*x_iter)->get_name() << endl;
+          indent_down();
+        } else {
+          f_service_ << indent() << "pass" << endl;
+        }
       }
 
       f_service_ << indent() << "except Exception as ex:" << endl
@@ -1989,7 +2020,8 @@ void t_py_generator::generate_deserialize_container(ofstream& out, t_type* ttype
 
   // For loop iterates over elements
   string i = tmp("_i");
-  indent(out) << "for " << i << " in xrange(" << size << "):" << endl;
+  indent(out) <<
+    "for " << i << " in range(" << size << "):" << endl;
 
   indent_up();
 

contrib/Vagrantfile

@@ -46,7 +46,7 @@ sudo apt-get install -qq libboost-dev libboost-test-dev libboost-program-options
 sudo apt-get install -qq ant openjdk-7-jdk maven
 
 # Python dependencies
-sudo apt-get install -qq python-all python-all-dev python-all-dbg python-setuptools python-support
+sudo apt-get install -qq python-all python-all-dev python-all-dbg python-setuptools python-support python-six
 
 # Ruby dependencies
 sudo apt-get install -qq ruby ruby-dev

lib/py/setup.py

@@ -19,6 +19,7 @@
 # under the License.
 #
 
+import platform
 import sys
 try:
     from setuptools import setup, Extension
@@ -69,14 +70,15 @@ def run_setup(with_binary):
         )
     else:
         extensions = dict()
-        
+
     setup(name = 'thrift',
         version = '1.0.0-dev',
         description = 'Python bindings for the Apache Thrift RPC system',
         author = 'Thrift Developers',
         author_email = 'dev@thrift.apache.org',
         url = 'http://thrift.apache.org',
         license = 'Apache License 2.0',
+        install_requires=['six>=1.7.2'],
         packages = [
             'thrift',
             'thrift.protocol',
@@ -90,15 +92,20 @@ def run_setup(with_binary):
             'Intended Audience :: Developers',
             'Programming Language :: Python',
             'Programming Language :: Python :: 2',
+            'Programming Language :: Python :: 3',
             'Topic :: Software Development :: Libraries',
             'Topic :: System :: Networking'
         ],
-        use_2to3 = True,
         **extensions
     )
 
 try:
-    run_setup(True)
+    with_binary = False
+    # Don't even try to build the C module unless we're on CPython 2.x.
+    # TODO: fix it for CPython 3.x
+    if platform.python_implementation() == 'CPython' and sys.version_info < (3,):
+        with_binary = True
+    run_setup(with_binary)
 except BuildFailed:
     print()
     print('*' * 80)

lib/py/src/TSCons.py

@@ -19,6 +19,8 @@
 
 from os import path
 from SCons.Builder import Builder
+from six.moves import map
+from six.moves import zip
 
 
 def scons_env(env, add=''):

lib/py/src/TSerialization.py

@@ -17,8 +17,8 @@
 # under the License.
 #
 
-from protocol import TBinaryProtocol
-from transport import TTransport
+from .protocol import TBinaryProtocol
+from .transport import TTransport
 
 
 def serialize(thrift_object,

lib/py/src/TTornado.py

@@ -20,11 +20,10 @@
 from __future__ import absolute_import
 import socket
 import struct
-
 import logging
 logger = logging.getLogger(__name__)
 
-from thrift.transport.TTransport import TTransportException, TTransportBase, TMemoryBuffer
+from .transport.TTransport import TTransportException, TTransportBase, TMemoryBuffer
 
 from io import BytesIO
 from collections import deque

lib/py/src/compat.py

@@ -0,0 +1,27 @@
+import sys
+
+if sys.version_info[0] == 2:
+
+  from cStringIO import StringIO as BufferIO
+
+  def binary_to_str(bin_val):
+    return bin_val
+
+  def str_to_binary(str_val):
+    return str_val
+
+else:
+
+  from io import BytesIO as BufferIO
+
+  def binary_to_str(bin_val):
+    try:
+      return bin_val.decode('utf8')
+    except:
+      return bin_val
+
+  def str_to_binary(str_val):
+    try:
+      return bytearray(str_val, 'utf8')
+    except:
+      return str_val

lib/py/src/protocol/TBase.py

@@ -17,7 +17,6 @@
 # under the License.
 #
 
-from thrift.Thrift import *
 from thrift.protocol import TBinaryProtocol
 from thrift.transport import TTransport
 
@@ -31,8 +30,7 @@ class TBase(object):
   __slots__ = []
 
   def __repr__(self):
-    L = ['%s=%r' % (key, getattr(self, key))
-              for key in self.__slots__]
+    L = ['%s=%r' % (key, getattr(self, key)) for key in self.__slots__]
     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
 
   def __eq__(self, other):
@@ -50,9 +48,9 @@ def __ne__(self, other):
 
   def read(self, iprot):
     if (iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and
-        isinstance(iprot.trans, TTransport.CReadableTransport) and
-        self.thrift_spec is not None and
-        fastbinary is not None):
+       isinstance(iprot.trans, TTransport.CReadableTransport) and
+       self.thrift_spec is not None and
+       fastbinary is not None):
       fastbinary.decode_binary(self,
                                iprot.trans,
                                (self.__class__, self.thrift_spec))
@@ -61,21 +59,13 @@ def read(self, iprot):
 
   def write(self, oprot):
     if (oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and
-        self.thrift_spec is not None and
-        fastbinary is not None):
+       self.thrift_spec is not None and
+       fastbinary is not None):
       oprot.trans.write(
         fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
       return
     oprot.writeStruct(self, self.thrift_spec)
 
 
-class TExceptionBase(Exception):
-  # old style class so python2.4 can raise exceptions derived from this
-  #  This can't inherit from TBase because of that limitation.
+class TExceptionBase(TBase, Exception):
   __slots__ = []
-
-  __repr__ = TBase.__repr__.im_func
-  __eq__ = TBase.__eq__.im_func
-  __ne__ = TBase.__ne__.im_func
-  read = TBase.read.im_func
-  write = TBase.write.im_func