hg merge default

--HG--
branch : ffistruct

pypy/doc/test/test_whatsnew.py

@@ -0,0 +1,80 @@
+import py
+import pypy
+from commands import getoutput
+ROOT = py.path.local(pypy.__file__).dirpath().dirpath()
+
+
+def parse_doc(s):
+    startrev = None
+    branches = set()
+    def parseline(line):
+        _, value = line.split(':', 1)
+        return value.strip()
+    #
+    for line in s.splitlines():
+        if line.startswith('.. startrev:'):
+            startrev = parseline(line)
+        elif line.startswith('.. branch:'):
+            branches.add(parseline(line))
+    return startrev, branches
+
+def get_merged_branches(path, startrev, endrev):
+    # X = take all the merges which are descendants of startrev and are on default
+    # revset = all the parents of X which are not on default
+    # ===>
+    # revset contains all the branches which have been merged to default since
+    # startrev
+    revset = 'parents(%s::%s and \
+                      merge() and \
+                      branch(default)) and \
+              not branch(default)' % (startrev, endrev)
+    cmd = r"hg log -R '%s' -r '%s' --template '{branches}\n'" % (path, revset)
+    out = getoutput(cmd)
+    branches = set(map(str.strip, out.splitlines()))
+    return branches
+
+
+def test_parse_doc():
+    s = """
+=====
+Title
+=====
+
+.. startrev: 12345
+
+bla bla bla bla
+
+.. branch: foobar
+
+xxx yyy zzz
+
+.. branch: hello
+
+qqq www ttt
+"""
+    startrev, branches = parse_doc(s)
+    assert startrev == '12345'
+    assert branches == set(['foobar', 'hello'])
+
+def test_get_merged_branches():
+    branches = get_merged_branches(ROOT, 'f34f0c11299f', '79770e0c2f93')
+    assert branches == set(['numpy-indexing-by-arrays-bool',
+                            'better-jit-hooks-2',
+                            'numpypy-ufuncs'])
+
+def test_whatsnew():
+    doc = ROOT.join('pypy', 'doc')
+    whatsnew_list = doc.listdir('whatsnew-*.rst')
+    whatsnew_list.sort()
+    last_whatsnew = whatsnew_list[-1].read()
+    startrev, documented = parse_doc(last_whatsnew)
+    merged = get_merged_branches(ROOT, startrev, '')
+    not_documented = merged.difference(documented)
+    not_merged = documented.difference(merged)
+    print 'Branches merged but not documented:'
+    print '\n'.join(not_documented)
+    print
+    print 'Branches documented but not merged:'
+    print '\n'.join(not_merged)
+    print
+    assert not not_documented and not not_merged

pypy/doc/whatsnew-1.9.rst

@@ -0,0 +1,72 @@
+======================
+What's new in PyPy 1.9
+======================
+
+.. this is the revision just after the creation of the release-1.8.x branch
+.. startrev: a4261375b359
+
+.. branch: array_equal
+.. branch: better-jit-hooks-2
+.. branch: exception-cannot-occur
+.. branch: faster-heapcache
+.. branch: faster-str-decode-escape
+.. branch: float-bytes
+Added some pritives for dealing with floats as raw bytes.
+.. branch: float-bytes-2
+Added more float byte primitives.
+.. branch: jit-frame-counter
+Put more debug info into resops.
+.. branch: kill-geninterp
+.. branch: kqueue
+Finished select.kqueue.
+.. branch: kwargsdict-strategy
+.. branch: matrixmath-dot
+numpypy can now handle matrix multiplication.
+.. branch: merge-2.7.2
+The stdlib was updated to version 2.7.2
+.. branch: ndmin
+.. branch: newindex
+.. branch: non-null-threadstate
+cpyext: Better support for PyEval_SaveThread and other PyTreadState_*
+functions.
+.. branch: numppy-flatitter
+.. branch: numpy-back-to-applevel
+.. branch: numpy-concatenate
+.. branch: numpy-indexing-by-arrays-bool
+.. branch: numpy-record-dtypes
+.. branch: numpy-single-jitdriver
+.. branch: numpy-ufuncs2
+.. branch: numpy-ufuncs3
+.. branch: numpypy-issue1137
+.. branch: numpypy-out
+The "out" argument was added to most of the numypypy functions.
+.. branch: numpypy-shape-bug
+.. branch: numpypy-ufuncs
+.. branch: pytest
+.. branch: safe-getargs-freelist
+.. branch: set-strategies
+.. branch: speedup-list-comprehension
+.. branch: stdlib-unification
+The directory "lib-python/modified-2.7" has been removed, and its
+content merged into "lib-python/2.7".
+.. branch: step-one-xrange
+The common case of a xrange iterator with no step argument specifed
+was somewhat optimized. The tightest loop involving it,
+sum(xrange(n)), is now 18% faster on average.
+.. branch: string-NUL
+PyPy refuses filenames with chr(0) characters. This is implemented in
+RPython which can enforce no-NUL correctness and propagation, similar
+to const-correctness in C++.
+.. branch: win32-cleanup
+.. branch: win32-cleanup2
+.. branch: win32-cleanup_2
+Many bugs were corrected for windows 32 bit. New functionality was added to
+test validity of file descriptors, leading to the removal of the  global 
+_invalid_parameter_handler
+.. branch: win64-stage1
+.. branch: zlib-mem-pressure
+
+
+.. "uninteresting" branches that we should just ignore for the whatsnew:
+.. branch: sanitize-finally-stack
+.. branch: revive-dlltool (preliminary work for sepcomp)

pypy/module/select/interp_kqueue.py

@@ -3,6 +3,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, generic_new_descr, GetSetProperty
 from pypy.rlib._rsocket_rffi import socketclose
+from pypy.rlib.rarithmetic import r_uint
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -226,9 +227,12 @@ def __del__(self):
         if self.event:
             lltype.free(self.event, flavor="raw")
 
-    @unwrap_spec(filter=int, flags='c_uint', fflags='c_uint', data=int, udata='c_uint')
-    def descr__init__(self, space, w_ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0):
-        ident = space.c_filedescriptor_w(w_ident)
+    @unwrap_spec(filter=int, flags='c_uint', fflags='c_uint', data=int, udata=r_uint)
+    def descr__init__(self, space, w_ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=r_uint(0)):
+        if space.isinstance_w(w_ident, space.w_long):
+            ident = space.uint_w(w_ident)
+        else:
+            ident = r_uint(space.c_filedescriptor_w(w_ident))
 
         self.event = lltype.malloc(kevent, flavor="raw")
         rffi.setintfield(self.event, "c_ident", ident)
@@ -320,7 +324,7 @@ def descr_get_data(self, space):
         return space.wrap(self.event.c_data)
 
     def descr_get_udata(self, space):
-        return space.wrap(rffi.cast(rffi.SIZE_T, self.event.c_udata))
+        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.event.c_udata))
 
 
 W_Kevent.typedef = TypeDef("select.kevent",

pypy/module/select/interp_select.py

@@ -74,6 +74,32 @@ def poll(self, space, w_timeout=None):
     pollmethods[methodname] = interp2app(getattr(Poll, methodname))
 Poll.typedef = TypeDef('select.poll', **pollmethods)
 
+# ____________________________________________________________
+
+
+from pypy.rlib import _rsocket_rffi as _c
+from pypy.rpython.lltypesystem import lltype, rffi
+
+
+def _build_fd_set(space, list_w, ll_list, nfds):
+    _c.FD_ZERO(ll_list)
+    fdlist = []
+    for w_f in list_w:
+        fd = space.c_filedescriptor_w(w_f)
+        if fd > nfds:
+            nfds = fd
+        _c.FD_SET(fd, ll_list)
+        fdlist.append(fd)
+    return fdlist, nfds
+_build_fd_set._always_inline_ = True    # get rid of the tuple result
+
+def _unbuild_fd_set(space, list_w, fdlist, ll_list, reslist_w):
+    for i in range(len(fdlist)):
+        fd = fdlist[i]
+        if _c.FD_ISSET(fd, ll_list):
+            reslist_w.append(list_w[i])
+
+
 def select(space, w_iwtd, w_owtd, w_ewtd, w_timeout=None):
     """Wait until one or more file descriptors are ready for some kind of I/O.
 The first three arguments are sequences of file descriptors to be waited for:
@@ -99,29 +125,62 @@ def select(space, w_iwtd, w_owtd, w_ewtd, w_timeout=None):
     iwtd_w = space.listview(w_iwtd)
     owtd_w = space.listview(w_owtd)
     ewtd_w = space.listview(w_ewtd)
-    iwtd = [space.c_filedescriptor_w(w_f) for w_f in iwtd_w]
-    owtd = [space.c_filedescriptor_w(w_f) for w_f in owtd_w]
-    ewtd = [space.c_filedescriptor_w(w_f) for w_f in ewtd_w]
-    iwtd_d = {}
-    owtd_d = {}
-    ewtd_d = {}
-    for i in range(len(iwtd)):
-        iwtd_d[iwtd[i]] = iwtd_w[i]
-    for i in range(len(owtd)):
-        owtd_d[owtd[i]] = owtd_w[i]
-    for i in range(len(ewtd)):
-        ewtd_d[ewtd[i]] = ewtd_w[i]
+
+    ll_inl  = lltype.nullptr(_c.fd_set.TO)
+    ll_outl = lltype.nullptr(_c.fd_set.TO)
+    ll_errl = lltype.nullptr(_c.fd_set.TO)
+    ll_timeval = lltype.nullptr(_c.timeval)
+
     try:
+        fdlistin  = None
+        fdlistout = None
+        fdlisterr = None
+        nfds = -1
+        if len(iwtd_w) > 0:
+            ll_inl = lltype.malloc(_c.fd_set.TO, flavor='raw')
+            fdlistin, nfds = _build_fd_set(space, iwtd_w, ll_inl, nfds)
+        if len(owtd_w) > 0:
+            ll_outl = lltype.malloc(_c.fd_set.TO, flavor='raw')
+            fdlistout, nfds = _build_fd_set(space, owtd_w, ll_outl, nfds)
+        if len(ewtd_w) > 0:
+            ll_errl = lltype.malloc(_c.fd_set.TO, flavor='raw')
+            fdlisterr, nfds = _build_fd_set(space, ewtd_w, ll_errl, nfds)
+
         if space.is_w(w_timeout, space.w_None):
-            iwtd, owtd, ewtd = rpoll.select(iwtd, owtd, ewtd)
+            timeout = -1.0
         else:
-            iwtd, owtd, ewtd = rpoll.select(iwtd, owtd, ewtd, space.float_w(w_timeout))
-    except rpoll.SelectError, s:
-        w_errortype = space.fromcache(Cache).w_error
-        raise OperationError(w_errortype, space.newtuple([
-            space.wrap(s.errno), space.wrap(s.get_msg())]))
-
-    return space.newtuple([
-        space.newlist([iwtd_d[i] for i in iwtd]),
-        space.newlist([owtd_d[i] for i in owtd]),
-        space.newlist([ewtd_d[i] for i in ewtd])])
+            timeout = space.float_w(w_timeout)
+        if timeout >= 0.0:
+            ll_timeval = rffi.make(_c.timeval)
+            i = int(timeout)
+            rffi.setintfield(ll_timeval, 'c_tv_sec', i)
+            rffi.setintfield(ll_timeval, 'c_tv_usec', int((timeout-i)*1000000))
+
+        res = _c.select(nfds + 1, ll_inl, ll_outl, ll_errl, ll_timeval)
+
+        if res < 0:
+            errno = _c.geterrno()
+            msg = _c.socket_strerror_str(errno)
+            w_errortype = space.fromcache(Cache).w_error
+            raise OperationError(w_errortype, space.newtuple([
+                space.wrap(errno), space.wrap(msg)]))
+
+        resin_w = []
+        resout_w = []
+        reserr_w = []
+        if res > 0:
+            if fdlistin is not None:
+                _unbuild_fd_set(space, iwtd_w, fdlistin,  ll_inl,  resin_w)
+            if fdlistout is not None:
+                _unbuild_fd_set(space, owtd_w, fdlistout, ll_outl, resout_w)
+            if fdlisterr is not None:
+                _unbuild_fd_set(space, ewtd_w, fdlisterr, ll_errl, reserr_w)
+    finally:
+        if ll_timeval: lltype.free(ll_timeval, flavor='raw')
+        if ll_errl:    lltype.free(ll_errl, flavor='raw')
+        if ll_outl:    lltype.free(ll_outl, flavor='raw')
+        if ll_inl:     lltype.free(ll_inl, flavor='raw')
+
+    return space.newtuple([space.newlist(resin_w),
+                           space.newlist(resout_w),
+                           space.newlist(reserr_w)])

pypy/module/select/test/test_kqueue.py

@@ -100,7 +100,7 @@ def test_queue_event(self):
         client.setblocking(False)
         try:
             client.connect(("127.0.0.1", server_socket.getsockname()[1]))
-        except socket.error as e:
+        except socket.error, e:
             if 'bsd' in sys.platform:
                 assert e.args[0] == errno.ENOENT
             else:

pypy/module/signal/interp_signal.py

@@ -16,19 +16,27 @@
 def setup():
     for key, value in cpy_signal.__dict__.items():
         if (key.startswith('SIG') or key.startswith('CTRL_')) and \
-                is_valid_int(value):
+                is_valid_int(value) and \
+                key != 'SIG_DFL' and key != 'SIG_IGN':
             globals()[key] = value
             yield key
 
 NSIG    = cpy_signal.NSIG
 SIG_DFL = cpy_signal.SIG_DFL
 SIG_IGN = cpy_signal.SIG_IGN
 signal_names = list(setup())
-signal_values = [globals()[key] for key in signal_names]
 signal_values = {}
 for key in signal_names:
     signal_values[globals()[key]] = None
-
+if sys.platform == 'win32' and not hasattr(cpy_signal,'CTRL_C_EVENT'):
+    # XXX Hack to revive values that went missing,
+    #     Remove this once we are sure the host cpy module has them.
+    signal_values[0] = None
+    signal_values[1] = None
+    signal_names.append('CTRL_C_EVENT')
+    signal_names.append('CTRL_BREAK_EVENT')
+    CTRL_C_EVENT = 0
+    CTRL_BREAK_EVENT = 1
 includes = ['stdlib.h', 'src/signals.h']
 if sys.platform != 'win32':
     includes.append('sys/time.h')

pypy/module/signal/test/test_signal.py

@@ -43,7 +43,11 @@ def setup_class(cls):
         cls.w_signal = space.appexec([], "(): import signal; return signal")
 
     def test_exported_names(self):
+        import os
         self.signal.__dict__   # crashes if the interpleveldefs are invalid
+        if os.name == 'nt':
+            assert self.signal.CTRL_BREAK_EVENT == 1
+            assert self.signal.CTRL_C_EVENT == 0
 
     def test_basics(self):
         import types, os