Merge pull request #494 from minrk/configure-pypy

support configure and bundled libzmq on pypy

.travis.yml

@@ -14,10 +14,6 @@ before_install:
   - sudo add-apt-repository -y ppa:shnatsel/dnscrypt
   - sudo apt-get update
   - if [[ $ZMQ != 'bundled' ]]; then sudo apt-get install -qq libzmq3-dev libsodium-dev; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then sudo add-apt-repository -y ppa:pypy/ppa || true; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then sudo apt-get -y update && sudo apt-get -y install pypy; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then virtualenv -p /usr/bin/pypy pypy && source pypy/bin/activate; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then pip install -q --use-mirrors py cffi; fi
   - if [[ $TRAVIS_PYTHON_VERSION != 'pypy' ]]; then pip install -q --use-mirrors cython --install-option='--no-cython-compile'; fi
 
   - if [[ $ZMQ == 'master' ]]; then git clone --depth 1 https://github.com/zeromq/libzmq; fi
@@ -30,8 +26,6 @@ install:
 
 matrix:
   exclude:
-    - python: pypy
-      env: ZMQ=bundled
     - python: 2.6
       env: ZMQ=bundled
     - python: 2.6

setup.py

@@ -77,6 +77,8 @@
 # Flags
 #-----------------------------------------------------------------------------
 
+pypy = 'PyPy' in sys.version
+
 # reference points for zmq compatibility
 min_zmq = (2,1,4)
 target_zmq = bundled_version
@@ -255,11 +257,10 @@ def finalize_options(self):
     def save_config(self, name, cfg):
         """write config to JSON"""
         save_config(name, cfg, self.build_base)
+        save_config(name, cfg, os.path.join('zmq', 'utils'))
 
     def init_settings_from_config(self):
         """set up compiler settings, based on config"""
-        if 'PyPy' in sys.version:
-            self.compiler_settings = {}
         cfg = self.config
 
         if cfg['libzmq_extension']:
@@ -406,8 +407,6 @@ def check_zmq_version(self):
 
     def bundle_libsodium_extension(self, libzmq):
         bundledir = "bundled"
-        if "PyPy" in sys.version:
-            fatal("Can't bundle libsodium as an Extension in PyPy (yet!)")
         ext_modules = self.distribution.ext_modules
         if ext_modules and any(m.name == 'zmq.libsodium' for m in ext_modules):
             # I've already been run
@@ -463,8 +462,6 @@ def bundle_libsodium_extension(self, libzmq):
 
     def bundle_libzmq_extension(self):
         bundledir = "bundled"
-        if "PyPy" in sys.version:
-            fatal("Can't bundle libzmq as an Extension in PyPy (yet!)")
         ext_modules = self.distribution.ext_modules
         if ext_modules and any(m.name == 'zmq.libzmq' for m in ext_modules):
             # I've already been run
@@ -516,9 +513,19 @@ def bundle_libzmq_extension(self):
             # check if we need to link against Realtime Extensions library
             cc = new_compiler(compiler=self.compiler_type)
             cc.output_dir = self.build_temp
-            if not sys.platform.startswith(('darwin', 'freebsd')) \
-                and not cc.has_function('timer_create'):
+            if not sys.platform.startswith(('darwin', 'freebsd')):
+                line()
+                info("checking for timer_create")
+                if not cc.has_function('timer_create'):
+                    info("no timer_create, linking librt")
                     libzmq.libraries.append('rt')
+                else:
+                    info("ok")
+
+                if pypy:
+                    # seem to need explicit libstdc++ on linux + pypy
+                    # not sure why
+                    libzmq.libraries.append("stdc++")
 
             # On non-Windows, also bundle libsodium:
             self.bundle_libsodium_extension(libzmq)
@@ -605,16 +612,13 @@ def test_build(self, prefix, settings):
 
         return detected
 
-    
+
     def finish_run(self):
         self.save_config('config', self.config)
         line()
 
     def run(self):
         cfg = self.config
-        if 'PyPy' in sys.version:
-            info("PyPy: Nothing to configure")
-            return
 
         if cfg['libzmq_extension']:
             self.bundle_libzmq_extension()
@@ -1046,19 +1050,44 @@ def run(self):
             ext.sources = sources
         extensions.append(ext)
 
-if 'PyPy' in sys.version:
-    try:
-        from zmq.backend.cffi import ffi
-    except ImportError:
-        warn("Couldn't get CFFI extension")
-        extensions = []
-    else:
-        extensions = [ffi.verifier.get_extension()]
+if pypy:
+    # add dummy extension, to ensure build_ext runs
+    dummy_ext = Extension('dummy', sources=[])
+    extensions = [dummy_ext]
+
+    bld_ext = cmdclass['build_ext']
+    class pypy_build_ext(bld_ext):
+        """hack to build pypy extension only after building bundled libzmq
+        
+        otherwise it will fail when libzmq is bundled.
+        """
+        def build_extensions(self):
+            self.extensions.remove(dummy_ext)
+            bld_ext.build_extensions(self)
+            # build ffi extension after bundled libzmq,
+            # because it may depend on linking it
+            here = os.getcwd()
+            sys.path.insert(0, self.build_lib)
+            try:
+                from zmq.backend.cffi import ffi
+            except ImportError as e:
+                warn("Couldn't get CFFI extension: %s" % e)
+            else:
+                ext = ffi.verifier.get_extension()
+                self.extensions.append(ext)
+                self.build_extension(ext)
+            finally:
+                sys.path.pop(0)
+
+
+    # How many build_ext subclasses is this? 5? Gross.
+    cmdclass['build_ext'] = pypy_build_ext
+
 
-package_data = {'zmq':['*.pxd'],
-                'zmq.backend.cython':['*.pxd'],
-                'zmq.devices':['*.pxd'],
-                'zmq.utils':['*.pxd', '*.h'],
+package_data = {'zmq': ['*.pxd'],
+                'zmq.backend.cython': ['*.pxd'],
+                'zmq.devices': ['*.pxd'],
+                'zmq.utils': ['*.pxd', '*.h', '*.json'],
 }
 
 package_data['zmq'].append('libzmq'+lib_ext)
@@ -1130,7 +1159,7 @@ def find_packages():
         'Programming Language :: Python :: 3.3',
     ],
 )
-if 'setuptools' in sys.modules and 'PyPy' in sys.version:
+if 'setuptools' in sys.modules and pypy:
     setup_args['install_requires'] = [
         'py',
         'cffi',

zmq/backend/cffi/_cffi.py

@@ -14,10 +14,11 @@
 # Imports
 #-----------------------------------------------------------------------------
 
+import json
 import os
+from os.path import dirname, join
 from cffi import FFI
 
-import zmq.utils
 from zmq.utils.constant_names import all_names, no_prefix
 
 
@@ -116,11 +117,47 @@
 int get_ipc_path_max_len(void);
 '''
 
+def load_compiler_config():
+    import zmq
+    zmq_dir = dirname(zmq.__file__)
+    zmq_parent = dirname(zmq_dir)
+
+    fname = join(zmq_dir, 'utils', 'compiler.json')
+    if os.path.exists(fname):
+        with open(fname) as f:
+            cfg = json.load(f)
+    else:
+        cfg = {}
+
+    cfg.setdefault("include_dirs", [])
+    cfg.setdefault("library_dirs", [])
+    cfg.setdefault("runtime_library_dirs", [])
+    cfg.setdefault("libraries", ["zmq"])
+
+    # cast to str, because cffi can't handle unicode paths (?!)
+    cfg['libraries'] = [str(lib) for lib in cfg['libraries']]
+    for key in ("include_dirs", "library_dirs", "runtime_library_dirs"):
+        # interpret paths relative to parent of zmq (like source tree)
+        abs_paths = []
+        for p in cfg[key]:
+            if p.startswith('zmq'):
+                p = join(zmq_parent, p)
+            abs_paths.append(str(p))
+        cfg[key] = abs_paths
+    return cfg
+
+cfg = load_compiler_config()
+
 def zmq_version_info():
     ffi_check = FFI()
     ffi_check.cdef('void zmq_version(int *major, int *minor, int *patch);')
+    cfg = load_compiler_config()
     C_check_version = ffi_check.verify('#include <zmq.h>',
-                                            libraries=['c', 'zmq'])
+        libraries=cfg['libraries'],
+        include_dirs=cfg['include_dirs'],
+        library_dirs=cfg['library_dirs'],
+        runtime_library_dirs=cfg['runtime_library_dirs'],
+    )
     major = ffi.new('int*')
     minor = ffi.new('int*')
     patch = ffi.new('int*')
@@ -169,7 +206,6 @@ def _make_defines(names):
 
 
 ffi.cdef(functions)
-zmq_utils = os.path.dirname(zmq.utils.__file__)
 
 C = ffi.verify('''
     #include <stdio.h>
@@ -185,7 +221,12 @@ def _make_defines(names):
     return sizeof(dummy->sun_path) - 1;
 }
 
-''', libraries=['c', 'zmq'], include_dirs=[zmq_utils])
+''',
+    libraries=cfg['libraries'],
+    include_dirs=cfg['include_dirs'],
+    library_dirs=cfg['library_dirs'],
+    runtime_library_dirs=cfg['runtime_library_dirs'],
+)
 
 nsp = new_sizet_pointer = lambda length: ffi.new('size_t*', length)