restore npy_PyFile_Dup api for 1.8.x

doc/release/1.8.1-notes.rst

@@ -47,3 +47,16 @@ Issues fixed
 * gh-4225: fix log1p and exmp1 return for np.inf on windows compiler builds
 * gh-4359: Fix infinite recursion in str.format of flex arrays
 * gh-4145: Incorrect shape of broadcast result with the exponent operator
+
+Deprecations
+============
+
+C-API
+~~~~~
+
+The utility function npy_PyFile_Dup and npy_PyFile_DupClose are broken by the
+internal buffering python 3 applies to its file objects.
+To fix this two new functions npy_PyFile_Dup2 and npy_PyFile_DupClose2 are
+declared in npy_3kcompat.h and the old functions are deprecated.
+Due to the fragile nature of these functions it is recommended to instead use
+the python API when possible.

numpy/core/include/numpy/npy_3kcompat.h

@@ -141,12 +141,11 @@ PyUnicode_Concat2(PyObject **left, PyObject *right)
  * PyFile_* compatibility
  */
 #if defined(NPY_PY3K)
-
 /*
  * Get a FILE* handle to the file represented by the Python object
  */
 static NPY_INLINE FILE*
-npy_PyFile_Dup(PyObject *file, char *mode, npy_off_t *orig_pos)
+npy_PyFile_Dup2(PyObject *file, char *mode, npy_off_t *orig_pos)
 {
     int fd, fd2;
     PyObject *ret, *os;
@@ -221,7 +220,7 @@ npy_PyFile_Dup(PyObject *file, char *mode, npy_off_t *orig_pos)
  * Close the dup-ed file handle, and seek the Python one to the current position
  */
 static NPY_INLINE int
-npy_PyFile_DupClose(PyObject *file, FILE* handle, npy_off_t orig_pos)
+npy_PyFile_DupClose2(PyObject *file, FILE* handle, npy_off_t orig_pos)
 {
     int fd;
     PyObject *ret;
@@ -269,10 +268,55 @@ npy_PyFile_Check(PyObject *file)
     return 1;
 }
 
+/*
+ * DEPRECATED DO NOT USE
+ * use npy_PyFile_Dup2 instead
+ * this function will mess ups python3 internal file object buffering
+ * Get a FILE* handle to the file represented by the Python object
+ */
+static NPY_INLINE FILE*
+npy_PyFile_Dup(PyObject *file, char *mode)
+{
+    npy_off_t orig;
+    if (DEPRECATE("npy_PyFile_Dup is deprecated, use "
+                  "npy_PyFile_Dup2") < 0) {
+        return NULL;
+    }
+
+    return npy_PyFile_Dup2(file, mode, &orig);
+}
+
+/*
+ * DEPRECATED DO NOT USE
+ * use npy_PyFile_DupClose2 instead
+ * this function will mess ups python3 internal file object buffering
+ * Close the dup-ed file handle, and seek the Python one to the current position
+ */
+static NPY_INLINE int
+npy_PyFile_DupClose(PyObject *file, FILE* handle)
+{
+    PyObject *ret;
+    Py_ssize_t position;
+    position = npy_ftell(handle);
+    fclose(handle);
+
+    ret = PyObject_CallMethod(file, "seek", NPY_SSIZE_T_PYFMT "i", position, 0);
+    if (ret == NULL) {
+        return -1;
+    }
+    Py_DECREF(ret);
+    return 0;
+}
+
+
 #else
 
-#define npy_PyFile_Dup(file, mode, orig_pos_p) PyFile_AsFile(file)
-#define npy_PyFile_DupClose(file, handle, orig_pos) (0)
+/* DEPRECATED DO NOT USE */
+#define npy_PyFile_Dup(file, mode) PyFile_AsFile(file)
+#define npy_PyFile_DupClose(file, handle) (0)
+/* use these */
+#define npy_PyFile_Dup2(file, mode, orig_pos_p) PyFile_AsFile(file)
+#define npy_PyFile_DupClose2(file, handle, orig_pos) (0)
 #define npy_PyFile_Check PyFile_Check
 
 #endif

numpy/core/src/multiarray/methods.c

@@ -588,7 +588,7 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds)
         own = 0;
     }
 
-    fd = npy_PyFile_Dup(file, "wb", &orig_pos);
+    fd = npy_PyFile_Dup2(file, "wb", &orig_pos);
     if (fd == NULL) {
         PyErr_SetString(PyExc_IOError,
                 "first argument must be a string or open file");
@@ -597,7 +597,7 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds)
     if (PyArray_ToFile(self, fd, sep, format) < 0) {
         goto fail;
     }
-    if (npy_PyFile_DupClose(file, fd, orig_pos) < 0) {
+    if (npy_PyFile_DupClose2(file, fd, orig_pos) < 0) {
         goto fail;
     }
     if (own && npy_PyFile_CloseFile(file) < 0) {

numpy/core/src/multiarray/multiarraymodule.c

@@ -1995,7 +1995,7 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds)
         Py_INCREF(file);
         own = 0;
     }
-    fp = npy_PyFile_Dup(file, "rb", &orig_pos);
+    fp = npy_PyFile_Dup2(file, "rb", &orig_pos);
     if (fp == NULL) {
         PyErr_SetString(PyExc_IOError,
                 "first argument must be an open file");
@@ -2007,7 +2007,7 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds)
     }
     ret = PyArray_FromFile(fp, type, (npy_intp) nin, sep);
 
-    if (npy_PyFile_DupClose(file, fp, orig_pos) < 0) {
+    if (npy_PyFile_DupClose2(file, fp, orig_pos) < 0) {
         goto fail;
     }
     if (own && npy_PyFile_CloseFile(file) < 0) {