Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Allow multi-iter objects to handle more iterators. #226

Closed
wants to merge 3 commits into from

5 participants

@walshb

Hi

This is some sort of fix for http://mail.scipy.org/pipermail/numpy-discussion/2011-July/057831.html.

(By the way, the test above runs pretty slowly because of constructing a big array from the inputs).

But after that, it seems to work.

Cheers

Ben

numpy/core/src/multiarray/iterators.c
@@ -1503,6 +1503,25 @@
return 0;
}
+void
+multiter_allociters(PyArrayMultiIterObject *multi, int n)
+{
+ int i;
+
+ multi->numiter = n;
+
+ if (n <= 0) {
+ multi->iters = NULL;
+ return;
+ }
+
+ multi->iters = PyArray_malloc(n * sizeof(PyArrayIterObject *));
@charris Owner
charris added a note

I think this needs an error check. And also everywhere multiiter_allociters is called.

@charris Owner
charris added a note

Also, the argument to PyArray_malloc should be size_t. You might want some error check for negative values somewhere down the line.

@mwiebe Owner
mwiebe added a note

The argument to PyArray_malloc should be ok, since the sizeof operator returns a size_t. The error check, and returning an error code from this function are definitely necessary though

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
numpy/core/src/multiarray/iterators.c
@@ -1503,6 +1503,25 @@
return 0;
}
+void
+multiter_allociters(PyArrayMultiIterObject *multi, int n)
@charris Owner
charris added a note

I suspect this shoud be static, as well as have an error return.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
@charris
Owner

Wouldn't it be simpler to just increase NPY_MAXARGS? I don't know if we want to spend too much time on this iterator as it should be replaced in the future with nditer.

@mwiebe
Owner

The email referenced indicated that some legacy code was using 16 million arguments, which would definitely need dynamic allocation to handle. Because this change is pretty simple, I think it's fine to apply once it's fixed up. Removing the NPY_MAXARGS limitation in nditer is slightly more involved, and I don't want to tackle that in C, it's more appropriate for C++.

A test which calls choose with 100 arguments or so should be added to this pull request, to demonstrate that it's working correctly.

@walshb

Well, finally got round to dealing with the remaining problems with this. Hope it comes in handy for someone...

Cheers

@travisbot

This pull request fails (merged f9811d5 into 7d225bb).

@njsmith
Owner

The failure there is just the memcpy/memmove bug, really this PR passed the tests when they were last run. (Though that was long enough ago that someone should probably re-run them before merging this.)

@charris
Owner

@walshb Sorry for the long hiatus. Could you rebase this?

@charris
Owner

@walshb Pretty much bitrotted at this point. The 1.8 branch will be made Aug 18.

@charris
Owner

Closing this on account of staleness. @seberg Sounds like choose could use the select treatment.

@charris charris closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Mar 5, 2012
  1. @walshb
Commits on Jun 24, 2012
  1. @walshb
  2. @walshb
This page is out of date. Refresh to see the latest.
View
7 numpy/core/include/numpy/ndarraytypes.h
@@ -1247,9 +1247,12 @@ typedef struct {
npy_intp index; /* current index */
int nd; /* number of dims */
npy_intp dimensions[NPY_MAXDIMS]; /* dimensions */
- PyArrayIterObject *iters[NPY_MAXARGS]; /* iterators */
+ PyArrayIterObject **iters; /* iterators */
} PyArrayMultiIterObject;
+int multiter_allociters(PyArrayMultiIterObject *multi, int n);
+
+
#define _PyMIT(m) ((PyArrayMultiIterObject *)(m))
#define PyArray_MultiIter_RESET(multi) do { \
int __npy_mi; \
@@ -1308,7 +1311,7 @@ typedef struct {
npy_intp index; /* current index */
int nd; /* number of dims */
npy_intp dimensions[NPY_MAXDIMS]; /* dimensions */
- PyArrayIterObject *iters[NPY_MAXDIMS]; /* index object
+ PyArrayIterObject **iters; /* index object
iterators */
PyArrayIterObject *ait; /* flat Iterator for
underlying array */
View
62 numpy/core/src/multiarray/iterators.c
@@ -1503,6 +1503,31 @@ PyArray_Broadcast(PyArrayMultiIterObject *mit)
return 0;
}
+int
+multiter_allociters(PyArrayMultiIterObject *multi, int n)
+{
+ int i;
+
+ multi->numiter = n;
+
+ if (n <= 0) {
+ multi->iters = NULL;
+ return 0;
+ }
+
+ multi->iters = PyArray_malloc(n * sizeof(PyArrayIterObject *));
+ if (multi->iters == NULL) {
+ PyErr_NoMemory();
+ return -1;
+ }
+
+ for (i = 0; i < n; i++) {
+ multi->iters[i] = NULL;
+ }
+
+ return 0;
+}
+
/*NUMPY_API
* Get MultiIterator from array of Python objects and any additional
*
@@ -1523,10 +1548,9 @@ PyArray_MultiIterFromObjects(PyObject **mps, int n, int nadd, ...)
int i, ntot, err=0;
ntot = n + nadd;
- if (ntot < 2 || ntot > NPY_MAXARGS) {
+ if (ntot < 2) {
PyErr_Format(PyExc_ValueError,
- "Need between 2 and (%d) " \
- "array objects (inclusive).", NPY_MAXARGS);
+ "Need at least 2 array objects.");
return NULL;
}
multi = PyArray_malloc(sizeof(PyArrayMultiIterObject));
@@ -1535,10 +1559,11 @@ PyArray_MultiIterFromObjects(PyObject **mps, int n, int nadd, ...)
}
PyObject_Init((PyObject *)multi, &PyArrayMultiIter_Type);
- for (i = 0; i < ntot; i++) {
- multi->iters[i] = NULL;
+ if (multiter_allociters(multi, ntot) < 0) {
+ /* error already set */
+ return NULL;
}
- multi->numiter = ntot;
+
multi->index = 0;
va_start(va, nadd);
@@ -1600,10 +1625,11 @@ PyArray_MultiIterNew(int n, ...)
}
PyObject_Init((PyObject *)multi, &PyArrayMultiIter_Type);
- for (i = 0; i < n; i++) {
- multi->iters[i] = NULL;
+ if (multiter_allociters(multi, n) < 0) {
+ /* error already set */
+ return NULL;
}
- multi->numiter = n;
+
multi->index = 0;
va_start(va, n);
@@ -1647,13 +1673,12 @@ arraymultiter_new(PyTypeObject *NPY_UNUSED(subtype), PyObject *args, PyObject *k
}
n = PyTuple_Size(args);
- if (n < 2 || n > NPY_MAXARGS) {
+ if (n < 2) {
if (PyErr_Occurred()) {
return NULL;
}
PyErr_Format(PyExc_ValueError,
- "Need at least two and fewer than (%d) " \
- "array objects.", NPY_MAXARGS);
+ "Need at least two array objects.");
return NULL;
}
@@ -1663,11 +1688,13 @@ arraymultiter_new(PyTypeObject *NPY_UNUSED(subtype), PyObject *args, PyObject *k
}
PyObject_Init((PyObject *)multi, &PyArrayMultiIter_Type);
- multi->numiter = n;
- multi->index = 0;
- for (i = 0; i < n; i++) {
- multi->iters[i] = NULL;
+ if (multiter_allociters(multi, n) < 0) {
+ /* error already set */
+ return NULL;
}
+
+ multi->index = 0;
+
for (i = 0; i < n; i++) {
arr = PyArray_FromAny(PyTuple_GET_ITEM(args, i), NULL, 0, 0, 0, NULL);
if (arr == NULL) {
@@ -1723,6 +1750,9 @@ arraymultiter_dealloc(PyArrayMultiIterObject *multi)
for (i = 0; i < multi->numiter; i++) {
Py_XDECREF(multi->iters[i]);
}
+
+ PyArray_free(multi->iters);
+
Py_TYPE(multi)->tp_free((PyObject *)multi);
}
View
41 numpy/core/src/multiarray/mapping.c
@@ -1910,6 +1910,11 @@ _nonzero_indices(PyObject *myBool, PyArrayIterObject **iters)
return -1;
}
nd = PyArray_NDIM(ba);
+
+ if (iters == NULL) {
+ return nd; /* just count iters needed */
+ }
+
for (j = 0; j < nd; j++) {
iters[j] = NULL;
}
@@ -1998,7 +2003,7 @@ _convert_obj(PyObject *obj, PyArrayIterObject **iter)
else if (PyArray_Check(obj) && PyArray_ISBOOL((PyArrayObject *)obj)) {
return _nonzero_indices(obj, iter);
}
- else {
+ else if (iter != NULL) {
indtype = PyArray_DescrFromType(NPY_INTP);
arr = PyArray_FromAny(obj, indtype, 0, 0, NPY_ARRAY_FORCECAST, NULL);
if (arr == NULL) {
@@ -2313,9 +2318,6 @@ PyArray_MapIterNew(PyObject *indexobj, int oned, int fancy)
if (mit == NULL) {
return NULL;
}
- for (i = 0; i < NPY_MAXDIMS; i++) {
- mit->iters[i] = NULL;
- }
mit->index = 0;
mit->ait = NULL;
mit->subspace = NULL;
@@ -2353,10 +2355,15 @@ PyArray_MapIterNew(PyObject *indexobj, int oned, int fancy)
/* convert all inputs to iterators */
if (PyArray_Check(indexobj) &&
(PyArray_TYPE((PyArrayObject *)indexobj) == NPY_BOOL)) {
- mit->numiter = _nonzero_indices(indexobj, mit->iters);
+ if (multiter_allociters((PyArrayMultiIterObject *)mit, _nonzero_indices(indexobj, NULL)) < 0) {
+ goto fail;
+ }
if (mit->numiter < 0) {
goto fail;
}
+ if (_nonzero_indices(indexobj, mit->iters) < 0) {
+ goto fail;
+ }
mit->nd = 1;
mit->dimensions[0] = mit->iters[0]->dims_m1[0]+1;
Py_DECREF(mit->indexobj);
@@ -2369,7 +2376,9 @@ PyArray_MapIterNew(PyObject *indexobj, int oned, int fancy)
}
}
else if (PyArray_Check(indexobj) || !PyTuple_Check(indexobj)) {
- mit->numiter = 1;
+ if (multiter_allociters((PyArrayMultiIterObject *)mit, 1) < 0) {
+ goto fail;
+ }
indtype = PyArray_DescrFromType(NPY_INTP);
arr = (PyArrayObject *)PyArray_FromAny(indexobj, indtype, 0, 0,
NPY_ARRAY_FORCECAST, NULL);
@@ -2393,7 +2402,7 @@ PyArray_MapIterNew(PyObject *indexobj, int oned, int fancy)
PyObject *obj;
PyArrayIterObject **iterp;
PyObject *new;
- int numiters, j, n2;
+ int numiters, j, n2, totniters;
/*
* Make a copy of the tuple -- we will be replacing
* index objects with 0's
@@ -2407,9 +2416,25 @@ PyArray_MapIterNew(PyObject *indexobj, int oned, int fancy)
started = 0;
nonindex = 0;
j = 0;
+ /* count iters */
+ totniters = 0;
for (i = 0; i < n; i++) {
obj = PyTuple_GET_ITEM(indexobj,i);
iterp = mit->iters + mit->numiter;
+ if ((numiters=_convert_obj(obj, NULL)) < 0) {
+ Py_DECREF(new);
+ goto fail;
+ }
+ totniters += numiters;
+ }
+ /* set up iters */
+ if (multiter_allociters((PyArrayMultiIterObject *)mit, totniters) < 0) {
+ goto fail;
+ }
+ totniters = 0;
+ for (i = 0; i < n; i++) {
+ obj = PyTuple_GET_ITEM(indexobj,i);
+ iterp = mit->iters + totniters;
if ((numiters=_convert_obj(obj, iterp)) < 0) {
Py_DECREF(new);
goto fail;
@@ -2419,7 +2444,7 @@ PyArray_MapIterNew(PyObject *indexobj, int oned, int fancy)
if (nonindex) {
mit->consec = 0;
}
- mit->numiter += numiters;
+ totniters += numiters;
if (numiters == 1) {
PyTuple_SET_ITEM(new,j++, PyInt_FromLong(0));
}
View
10 numpy/ma/tests/test_core.py
@@ -3144,6 +3144,16 @@ def test_choose(self):
assert_equal(chosen.mask, [1, 0, 0, 1])
+ def test_choose_big(self):
+ "Test choose with large number of choices"
+ choices = [[0, 1, 2, 3], [10, 11, 12, 13],
+ [20, 21, 22, 23], [30, 31, 32, 33]]
+ n = 100
+ morechoices = choices * n
+ chosen = choose([2, 3, 1, 0], morechoices)
+ assert_equal(chosen, array([20, 31, 12, 3]))
+
+
def test_choose_with_out(self):
"Test choose with an explicit out keyword"
choices = [[0, 1, 2, 3], [10, 11, 12, 13],
Something went wrong with that request. Please try again.