Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

REV: "ENH: Improved performance of PyArray_FromAny for sequences of array-like" #14109

Merged
merged 1 commit into from Jul 25, 2019
Merged

REV: "ENH: Improved performance of PyArray_FromAny for sequences of array-like" #14109

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
184 changes: 62 additions & 122 deletions numpy/core/src/multiarray/ctors.c
View file
Open in desktop
Expand Up @@ -422,10 +422,6 @@ copy_and_swap(void *dst, void *src, int itemsize, npy_intp numitems,
}
}

NPY_NO_EXPORT PyObject *
_array_from_array_like(PyObject *op, PyArray_Descr *requested_dtype,
npy_bool writeable, PyObject *context);

/*
* adapted from Numarray,
* a: destination array
Expand All @@ -446,6 +442,11 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s,
if (dst == NULL)
dst = a;

/*
* This code is to ensure that the sequence access below will
* return a lower-dimensional sequence.
*/

/* INCREF on entry DECREF on exit */
Py_INCREF(s);

Expand All @@ -471,11 +472,6 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s,
return 0;
}

/*
* This code is to ensure that the sequence access below will
* return a lower-dimensional sequence.
*/

if (dim > PyArray_NDIM(a)) {
PyErr_Format(PyExc_ValueError,
"setArrayFromSequence: sequence/array dimensions mismatch.");
Expand All @@ -486,27 +482,6 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s,
if (slen < 0) {
goto fail;
}
if (slen > 0) {
/* gh-13659: try __array__ before using s as a sequence */
PyObject *tmp = _array_from_array_like(s, /*dtype*/NULL, /*writeable*/0,
/*context*/NULL);
if (tmp == NULL) {
goto fail;
}
else if (tmp == Py_NotImplemented) {
Py_DECREF(tmp);
}
else {
int r = PyArray_CopyInto(dst, (PyArrayObject *)tmp);
Py_DECREF(tmp);
if (r < 0) {
goto fail;
}
Py_DECREF(s);
return 0;
}
}

/*
* Either the dimensions match, or the sequence has length 1 and can
* be broadcast to the destination.
Expand Down Expand Up @@ -1577,90 +1552,6 @@ _array_from_buffer_3118(PyObject *memoryview)

}


/*
* Attempts to extract an array from an array-like object.
*
* array-like is defined as either
*
* * an object implementing the PEP 3118 buffer interface;
* * an object with __array_struct__ or __array_interface__ attributes;
* * an object with an __array__ function.
*
* Returns Py_NotImplemented if a given object is not array-like;
* PyArrayObject* in case of success and NULL in case of failure.
*/
NPY_NO_EXPORT PyObject *
_array_from_array_like(PyObject *op, PyArray_Descr *requested_dtype,
npy_bool writeable, PyObject *context) {
PyObject* tmp;

/* If op supports the PEP 3118 buffer interface */
if (!PyBytes_Check(op) && !PyUnicode_Check(op)) {
PyObject *memoryview = PyMemoryView_FromObject(op);
if (memoryview == NULL) {
PyErr_Clear();
}
else {
tmp = _array_from_buffer_3118(memoryview);
Py_DECREF(memoryview);
if (tmp == NULL) {
return NULL;
}

if (writeable
&& PyArray_FailUnlessWriteable((PyArrayObject *) tmp, "PEP 3118 buffer") < 0) {
Py_DECREF(tmp);
return NULL;
}

return tmp;
}
}

/* If op supports the __array_struct__ or __array_interface__ interface */
tmp = PyArray_FromStructInterface(op);
if (tmp == NULL) {
return NULL;
}
if (tmp == Py_NotImplemented) {
tmp = PyArray_FromInterface(op);
if (tmp == NULL) {
return NULL;
}
}

/*
* If op supplies the __array__ function.
* The documentation says this should produce a copy, so
* we skip this method if writeable is true, because the intent
* of writeable is to modify the operand.
* XXX: If the implementation is wrong, and/or if actual
* usage requires this behave differently,
* this should be changed!
*/
if (!writeable && tmp == Py_NotImplemented) {
tmp = PyArray_FromArrayAttr(op, requested_dtype, context);
if (tmp == NULL) {
return NULL;
}
}

if (tmp != Py_NotImplemented) {
if (writeable
&& PyArray_FailUnlessWriteable((PyArrayObject *) tmp,
"array interface object") < 0) {
Py_DECREF(tmp);
return NULL;
}
return tmp;
}

Py_INCREF(Py_NotImplemented);
return Py_NotImplemented;
}


/*NUMPY_API
* Retrieves the array parameters for viewing/converting an arbitrary
* PyObject* to a NumPy array. This allows the "innate type and shape"
Expand Down Expand Up @@ -1768,20 +1659,69 @@ PyArray_GetArrayParamsFromObject(PyObject *op,
return 0;
}

/* If op is an array-like */
tmp = _array_from_array_like(op, requested_dtype, writeable, context);
/* If op supports the PEP 3118 buffer interface */
if (!PyBytes_Check(op) && !PyUnicode_Check(op)) {

PyObject *memoryview = PyMemoryView_FromObject(op);
if (memoryview == NULL) {
PyErr_Clear();
}
else {
PyObject *arr = _array_from_buffer_3118(memoryview);
Py_DECREF(memoryview);
if (arr == NULL) {
return -1;
}
if (writeable
&& PyArray_FailUnlessWriteable((PyArrayObject *)arr, "PEP 3118 buffer") < 0) {
Py_DECREF(arr);
return -1;
}
*out_arr = (PyArrayObject *)arr;
return 0;
}
}

/* If op supports the __array_struct__ or __array_interface__ interface */
tmp = PyArray_FromStructInterface(op);
if (tmp == NULL) {
return -1;
}
else if (tmp != Py_NotImplemented) {
*out_arr = (PyArrayObject*) tmp;
return 0;
if (tmp == Py_NotImplemented) {
tmp = PyArray_FromInterface(op);
if (tmp == NULL) {
return -1;
}
}
else {
Py_DECREF(Py_NotImplemented);
if (tmp != Py_NotImplemented) {
if (writeable
&& PyArray_FailUnlessWriteable((PyArrayObject *)tmp,
"array interface object") < 0) {
Py_DECREF(tmp);
return -1;
}
*out_arr = (PyArrayObject *)tmp;
return (*out_arr) == NULL ? -1 : 0;
}

/*
* If op supplies the __array__ function.
* The documentation says this should produce a copy, so
* we skip this method if writeable is true, because the intent
* of writeable is to modify the operand.
* XXX: If the implementation is wrong, and/or if actual
* usage requires this behave differently,
* this should be changed!
*/
if (!writeable) {
tmp = PyArray_FromArrayAttr(op, requested_dtype, context);
if (tmp != Py_NotImplemented) {
*out_arr = (PyArrayObject *)tmp;
return (*out_arr) == NULL ? -1 : 0;
}
}

/* Try to treat op as a list of lists or array-like objects. */
/* Try to treat op as a list of lists */
if (!writeable && PySequence_Check(op)) {
int check_it, stop_at_string, stop_at_tuple, is_object;
int type_num, type;
Expand Down
23 changes: 0 additions & 23 deletions numpy/core/tests/test_multiarray.py
View file
Open in desktop
Expand Up @@ -969,29 +969,6 @@ def test_sequence_long(self):
assert_equal(np.array([long(4), 2**80, long(4)]).dtype, object)
assert_equal(np.array([2**80, long(4)]).dtype, object)

def test_sequence_of_array_like(self):
class ArrayLike:
def __init__(self):
self.__array_interface__ = {
"shape": (42,),
"typestr": "<i1",
"data": bytes(42)
}

# Make sure __array_*__ is used instead of Sequence methods.
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nvm. Removed the test for now, because __iter__ is now used again (tripping this assertion).

def __iter__(self):
raise AssertionError("__iter__ was called")

def __getitem__(self, idx):
raise AssertionError("__getitem__ was called")

def __len__(self):
return 42

assert_equal(
np.array([ArrayLike()]),
np.zeros((1, 42), dtype=np.byte))

def test_non_sequence_sequence(self):
"""Should not segfault.

Expand Down
20 changes: 0 additions & 20 deletions numpy/core/tests/test_regression.py
View file
Open in desktop
Expand Up @@ -2454,23 +2454,3 @@ class T(object):
__array_interface__ = {}

np.array([T()])

def test_2d__array__shape(self):
class T(object):
def __array__(self):
return np.ndarray(shape=(0,0))

# Make sure __array__ is used instead of Sequence methods.
def __iter__(self):
return iter([])

def __getitem__(self, idx):
raise AssertionError("__getitem__ was called")

def __len__(self):
return 0


t = T()
#gh-13659, would raise in broadcasting [x=t for x in result]
np.array([t])