Closes issue #586. Updates handling of nan behaviour in numpy.lib.function_base.median.

doc/release/1.9.0-notes.rst

@@ -243,7 +243,12 @@ For example ``np.float_(2) * [1]`` will be an error in the future.
 C-API
 ~~~~~
 
-None
+The utility function npy_PyFile_Dup and npy_PyFile_DupClose are broken by the
+internal buffering python 3 applies to its file objects.
+To fix this two new functions npy_PyFile_Dup2 and npy_PyFile_DupClose2 are
+declared in npy_3kcompat.h and the old functions are deprecated.
+Due to the fragile nature of these functions it is recommended to instead use
+the python API when possible.
 
 
 New Features

doc/swig/numpy.i

@@ -2246,7 +2246,7 @@
   PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$1));
   PyArrayObject* array = (PyArrayObject*) obj;
 
-  if (!array || require_fortran(array)) SWIG_fail;
+  if (!array || !require_fortran(array)) SWIG_fail;
   $result = SWIG_Python_AppendOutput($result,obj);
 }
 
@@ -2270,7 +2270,7 @@
   PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$4));
   PyArrayObject* array = (PyArrayObject*) obj;
 
-  if (!array || require_fortran(array)) SWIG_fail;
+  if (!array || !require_fortran(array)) SWIG_fail;
   $result = SWIG_Python_AppendOutput($result,obj);
 }
 
@@ -2345,7 +2345,7 @@
   PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1));
   PyArrayObject* array = (PyArrayObject*) obj;
 
-  if (!array || require_fortran(array)) SWIG_fail;
+  if (!array || !require_fortran(array)) SWIG_fail;
   $result = SWIG_Python_AppendOutput($result,obj);
 }
 
@@ -2370,7 +2370,7 @@
   PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5));
   PyArrayObject* array = (PyArrayObject*) obj;
 
-  if (!array || require_fortran(array)) SWIG_fail;
+  if (!array || !require_fortran(array)) SWIG_fail;
   $result = SWIG_Python_AppendOutput($result,obj);
 }
 
@@ -2680,7 +2680,7 @@
   PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$1));
   PyArrayObject* array = (PyArrayObject*) obj;
 
-  if (!array || require_fortran(array)) SWIG_fail;
+  if (!array || !require_fortran(array)) SWIG_fail;
 
 %#ifdef SWIGPY_USE_CAPSULE
     PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap);
@@ -2717,7 +2717,7 @@
   PyObject* obj = PyArray_SimpleNewFromData(3, dims, DATA_TYPECODE, (void*)(*$4));
   PyArrayObject* array = (PyArrayObject*) obj;
 
-  if (!array || require_fortran(array)) SWIG_fail;
+  if (!array || !require_fortran(array)) SWIG_fail;
 
 %#ifdef SWIGPY_USE_CAPSULE
     PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap);
@@ -2831,7 +2831,7 @@
   PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1));
   PyArrayObject* array = (PyArrayObject*) obj;
 
-  if (!array || require_fortran(array)) SWIG_fail;
+  if (!array || !require_fortran(array)) SWIG_fail;
 
 %#ifdef SWIGPY_USE_CAPSULE
     PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap);
@@ -2869,7 +2869,7 @@
   PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5));
   PyArrayObject* array = (PyArrayObject*) obj;
 
-  if (!array || require_fortran(array)) SWIG_fail;
+  if (!array || !require_fortran(array)) SWIG_fail;
 
 %#ifdef SWIGPY_USE_CAPSULE
     PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap);
@@ -2983,7 +2983,7 @@
   PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1));
   PyArrayObject* array = (PyArrayObject*) obj;
 
-  if (!array || require_fortran(array)) SWIG_fail;
+  if (!array || !require_fortran(array)) SWIG_fail;
 
 %#ifdef SWIGPY_USE_CAPSULE
     PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap);
@@ -3021,7 +3021,7 @@
   PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5));
   PyArrayObject* array = (PyArrayObject*) obj;
 
-  if (!array || require_fortran(array)) SWIG_fail;
+  if (!array || !require_fortran(array)) SWIG_fail;
 
 %#ifdef SWIGPY_USE_CAPSULE
     PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap);

numpy/core/include/numpy/npy_3kcompat.h

@@ -141,12 +141,11 @@ PyUnicode_Concat2(PyObject **left, PyObject *right)
  * PyFile_* compatibility
  */
 #if defined(NPY_PY3K)
-
 /*
  * Get a FILE* handle to the file represented by the Python object
  */
 static NPY_INLINE FILE*
-npy_PyFile_Dup(PyObject *file, char *mode, npy_off_t *orig_pos)
+npy_PyFile_Dup2(PyObject *file, char *mode, npy_off_t *orig_pos)
 {
     int fd, fd2;
     PyObject *ret, *os;
@@ -221,7 +220,7 @@ npy_PyFile_Dup(PyObject *file, char *mode, npy_off_t *orig_pos)
  * Close the dup-ed file handle, and seek the Python one to the current position
  */
 static NPY_INLINE int
-npy_PyFile_DupClose(PyObject *file, FILE* handle, npy_off_t orig_pos)
+npy_PyFile_DupClose2(PyObject *file, FILE* handle, npy_off_t orig_pos)
 {
     int fd;
     PyObject *ret;
@@ -269,10 +268,55 @@ npy_PyFile_Check(PyObject *file)
     return 1;
 }
 
+/*
+ * DEPRECATED DO NOT USE
+ * use npy_PyFile_Dup2 instead
+ * this function will mess ups python3 internal file object buffering
+ * Get a FILE* handle to the file represented by the Python object
+ */
+static NPY_INLINE FILE*
+npy_PyFile_Dup(PyObject *file, char *mode)
+{
+    npy_off_t orig;
+    if (DEPRECATE("npy_PyFile_Dup is deprecated, use "
+                  "npy_PyFile_Dup2") < 0) {
+        return NULL;
+    }
+
+    return npy_PyFile_Dup2(file, mode, &orig);
+}
+
+/*
+ * DEPRECATED DO NOT USE
+ * use npy_PyFile_DupClose2 instead
+ * this function will mess ups python3 internal file object buffering
+ * Close the dup-ed file handle, and seek the Python one to the current position
+ */
+static NPY_INLINE int
+npy_PyFile_DupClose(PyObject *file, FILE* handle)
+{
+    PyObject *ret;
+    Py_ssize_t position;
+    position = npy_ftell(handle);
+    fclose(handle);
+
+    ret = PyObject_CallMethod(file, "seek", NPY_SSIZE_T_PYFMT "i", position, 0);
+    if (ret == NULL) {
+        return -1;
+    }
+    Py_DECREF(ret);
+    return 0;
+}
+
+
 #else
 
-#define npy_PyFile_Dup(file, mode, orig_pos_p) PyFile_AsFile(file)
-#define npy_PyFile_DupClose(file, handle, orig_pos) (0)
+/* DEPRECATED DO NOT USE */
+#define npy_PyFile_Dup(file, mode) PyFile_AsFile(file)
+#define npy_PyFile_DupClose(file, handle) (0)
+/* use these */
+#define npy_PyFile_Dup2(file, mode, orig_pos_p) PyFile_AsFile(file)
+#define npy_PyFile_DupClose2(file, handle, orig_pos) (0)
 #define npy_PyFile_Check PyFile_Check
 
 #endif

numpy/core/src/multiarray/methods.c

@@ -588,7 +588,7 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds)
         own = 0;
     }
 
-    fd = npy_PyFile_Dup(file, "wb", &orig_pos);
+    fd = npy_PyFile_Dup2(file, "wb", &orig_pos);
     if (fd == NULL) {
         PyErr_SetString(PyExc_IOError,
                 "first argument must be a string or open file");
@@ -597,7 +597,7 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds)
     if (PyArray_ToFile(self, fd, sep, format) < 0) {
         goto fail;
     }
-    if (npy_PyFile_DupClose(file, fd, orig_pos) < 0) {
+    if (npy_PyFile_DupClose2(file, fd, orig_pos) < 0) {
         goto fail;
     }
     if (own && npy_PyFile_CloseFile(file) < 0) {

numpy/core/src/multiarray/multiarraymodule.c

@@ -1999,7 +1999,7 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds)
         Py_INCREF(file);
         own = 0;
     }
-    fp = npy_PyFile_Dup(file, "rb", &orig_pos);
+    fp = npy_PyFile_Dup2(file, "rb", &orig_pos);
     if (fp == NULL) {
         PyErr_SetString(PyExc_IOError,
                 "first argument must be an open file");
@@ -2011,7 +2011,7 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds)
     }
     ret = PyArray_FromFile(fp, type, (npy_intp) nin, sep);
 
-    if (npy_PyFile_DupClose(file, fp, orig_pos) < 0) {
+    if (npy_PyFile_DupClose2(file, fp, orig_pos) < 0) {
         goto fail;
     }
     if (own && npy_PyFile_CloseFile(file) < 0) {

numpy/core/src/npysort/selection.c.src

@@ -69,7 +69,6 @@ static NPY_INLINE void store_pivot(npy_intp pivot, npy_intp kth,
  *         npy_uint, npy_long, npy_ulong, npy_longlong, npy_ulonglong,
  *         npy_ushort, npy_float, npy_double, npy_longdouble, npy_cfloat,
  *         npy_cdouble, npy_clongdouble#
- * #inexact = 0*11, 1*7#
  */
 
 static npy_intp
@@ -323,8 +322,7 @@ int
         store_pivot(kth, kth, pivots, npiv);
         return 0;
     }
-    else if (@inexact@ && kth == num - 1) {
-        /* useful to check if NaN present via partition(d, (x, -1)) */
+    else if (kth == num - 1) {
         npy_intp k;
         npy_intp maxidx = low;
         @type@ maxval = v[IDX(low)];

numpy/distutils/fcompiler/__init__.py

@@ -38,7 +38,7 @@
 from numpy.distutils.ccompiler import CCompiler, gen_lib_options
 from numpy.distutils import log
 from numpy.distutils.misc_util import is_string, all_strings, is_sequence, \
-    make_temp_file, get_shared_lib_extension, quote
+    make_temp_file, get_shared_lib_extension
 from numpy.distutils.environment import EnvironmentConfig
 from numpy.distutils.exec_command import find_executable
 from numpy.distutils.compat import get_exception
@@ -582,12 +582,12 @@ def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts):
                       % (self.__class__.__name__, src))
             extra_compile_args = self.extra_f90_compile_args or []
         if self.object_switch[-1]==' ':
-            o_args = [self.object_switch.strip(), quote(obj)]
+            o_args = [self.object_switch.strip(), obj]
         else:
-            o_args = [self.object_switch.strip() + quote(obj)]
+            o_args = [self.object_switch.strip()+obj]
 
         assert self.compile_switch.strip()
-        s_args = [self.compile_switch, quote(src)]
+        s_args = [self.compile_switch, src]
 
         if extra_compile_args:
             log.info('extra %s options: %r' \
@@ -659,7 +659,6 @@ def link(self, target_desc, objects,
             else:
                 ld_args = objects + self.objects
             ld_args = ld_args + lib_opts + o_args
-            ld_args = [quote(ld_arg) for ld_arg in ld_args]
             if debug:
                 ld_args[:0] = ['-g']
             if extra_preargs:
@@ -988,4 +987,3 @@ def get_f77flags(src):
 
 if __name__ == '__main__':
     show_fcompilers()
-

numpy/distutils/misc_util.py

@@ -14,11 +14,6 @@
 import distutils
 from distutils.errors import DistutilsError
 
-try:
-    from pipes import quote
-except ImportError:
-    from shlex import quote
-
 try:
     set
 except NameError:
@@ -36,8 +31,7 @@
            'get_script_files', 'get_lib_source_files', 'get_data_files',
            'dot_join', 'get_frame', 'minrelpath', 'njoin',
            'is_sequence', 'is_string', 'as_list', 'gpaths', 'get_language',
-           'quote_args', 'quote', 'get_build_architecture', 'get_info',
-           'get_pkg_info']
+           'quote_args', 'get_build_architecture', 'get_info', 'get_pkg_info']
 
 class InstallableLib(object):
     """

numpy/distutils/npy_pkg_config.py

@@ -10,8 +10,6 @@
 else:
     from configparser import ConfigParser, SafeConfigParser, NoOptionError
 
-from numpy.distutils.misc_util import quote
-
 __all__ = ['FormatError', 'PkgNotFound', 'LibraryInfo', 'VariableSet',
         'read_config', 'parse_flags']
 
@@ -58,7 +56,7 @@ def parse_flags(line):
         * 'ignored'
 
     """
-    lexer = shlex.shlex(line, posix=True)
+    lexer = shlex.shlex(line)
     lexer.whitespace_split = True
 
     d = {'include_dirs': [], 'library_dirs': [], 'libraries': [],
@@ -90,6 +88,8 @@ def next_token(t):
 
     return d
 
+def _escape_backslash(val):
+    return val.replace('\\', '\\\\')
 
 class LibraryInfo(object):
     """
@@ -147,11 +147,11 @@ def sections(self):
 
     def cflags(self, section="default"):
         val = self.vars.interpolate(self._sections[section]['cflags'])
-        return quote(val)
+        return _escape_backslash(val)
 
     def libs(self, section="default"):
         val = self.vars.interpolate(self._sections[section]['libs'])
-        return quote(val)
+        return _escape_backslash(val)
 
     def __str__(self):
         m = ['Name: %s' % self.name]
@@ -289,7 +289,7 @@ def parse_config(filename, dirs=None):
     vars = {}
     if config.has_section('variables'):
         for name, value in config.items("variables"):
-            vars[name] = quote(value)
+            vars[name] = _escape_backslash(value)
 
     # Parse "normal" sections
     secs = [s for s in config.sections() if not s in ['meta', 'variables']]
@@ -338,7 +338,7 @@ def _read_config(f):
                              (pkgname, meta["name"]))
 
         mod = sys.modules[pkgname]
-        vars["pkgdir"] = quote(os.path.dirname(mod.__file__))
+        vars["pkgdir"] = _escape_backslash(os.path.dirname(mod.__file__))
 
     return LibraryInfo(name=meta["name"], description=meta["description"],
             version=meta["version"], sections=sections, vars=VariableSet(vars))