Skip to content

Commit

Permalink
Single new array function
Browse files Browse the repository at this point in the history 
Unfortunately cannot use default args in pxd file; I also wouldn't know how the two array functions could be added as methods to the array class.
Restored "inline" attribute for array creation function, it can be disabled with a \#DEF
Removed some unused/redundant attributes.
Moved selected comments to docstrings.
Replaced tabs w/spaces.
  • Loading branch information
@andreasvc
andreasvc committed May 2, 2012
1 parent 8fc6e20 commit 13d9929
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 99 deletions.
1 change: 0 additions & 1 deletion Cython/Includes/cpython/__init__.pxd
View file
Expand Up @@ -164,7 +164,6 @@ from cpython.weakref cimport *
from cpython.getargs cimport *
from cpython.pythread cimport *
from cpython.pystate cimport *
from cpython.array cimport *

# Python <= 2.x
from cpython.cobject cimport *
Expand Down
81 changes: 32 additions & 49 deletions Cython/Includes/cpython/array.pxd
View file
Expand Up @@ -38,7 +38,7 @@
Suitable as lightweight arrays intra Cython without speed penalty.
Replacement for C stack/malloc arrays; no trouble with refcounting,
mem.leaks; seamless Python compatibility, buffer() optionA
mem.leaks; seamless Python compatibility, buffer() optional
IMPORTANT: arrayarray.h (arrayobject, arraydescr) is not part of
Expand All @@ -48,23 +48,17 @@
last changes: 2009-05-15 rk
: 2009-12-06 bp
: 2012-05-02 andreasvc
"""
from libc cimport stdlib
from libc.string cimport strcat, strncat, \
memset, memchr, memcmp, memcpy, memmove

cdef extern from "stdlib.h" nogil:
void *memset(void *str, int c, size_t n)
char *strcat(char *str1, char *str2)
char *strncat(char *str1, char *str2, size_t n)
void *memchr(void *str, int c, size_t n)
int memcmp(void *str1, void *str2, size_t n)
void *memcpy(void *str1, void *str2, size_t n)
void *memmove(void *str1, void *str2, size_t n)

from cpython.ref cimport PyTypeObject
from cpython.exc cimport PyErr_BadArgument

cdef extern from "arrayarray.h":
ctypedef void PyTypeObject
ctypedef short Py_UNICODE

This comment has been minimized.

Sign in to view

Copy link
@scoder

scoder May 6, 2012

This line isn't needed - Py_UNICODE is a built-in type in Cython.
int PyErr_BadArgument()
ctypedef class array.array [object arrayobject]
ctypedef object GETF(array a, Py_ssize_t ix)
ctypedef object SETF(array a, Py_ssize_t ix, object o)
Expand All @@ -79,17 +73,10 @@ cdef extern from "arrayarray.h":

cdef:
PyTypeObject* ob_type

int ob_size # number of valid items;
unsigned length # == ob_size (by union)

This comment has been minimized.

Sign in to view

Copy link
@scoder

scoder May 6, 2012

Py_ssize_t

char* ob_item # to first item

Py_ssize_t allocated # bytes
arraydescr* ob_descr # struct arraydescr *ob_descr;
object weakreflist # /* List of weak references */

# view's of ob_item:
# views of ob_item:
float* _f # direct float pointer access to buffer
double* _d # double ...
int* _i
Expand All @@ -112,7 +99,7 @@ cdef extern from "arrayarray.h":
cdef unsigned rows, columns, itemsize

info.suboffsets = NULL
info.buf = self.ob_item
info.buf = self._c
info.readonly = 0
info.ndim = 1
info.itemsize = itemsize = self.ob_descr.itemsize # e.g. sizeof(float)
Expand Down Expand Up @@ -144,44 +131,40 @@ cdef extern from "arrayarray.h":
int resize_smart(array self, Py_ssize_t n)


# fast creation of a new array - init with zeros
# yet you need a (any) template array of the same item type (but not same size)
cdef inline array zeros_like(array sametype):
cdef array op = newarrayobject(<PyTypeObject*>sametype.ob_type, sametype.ob_size, sametype.ob_descr)
if op:
memset(op.ob_item, 0, op.ob_size * op.ob_descr.itemsize)
return op

# fast creation of a new array - no init with zeros
cdef inline array new_array(array sametype, unsigned n):
return newarrayobject( <PyTypeObject*>sametype.ob_type, n, sametype.ob_descr)

# fast creation of a new array - no init with zeros, same length
cdef inline array empty_like(array sametype):
return newarrayobject(<PyTypeObject*>sametype.ob_type, sametype.op.ob_size,
sametype.ob_descr)
cdef inline array clone(array template, Py_ssize_t length, bint zero):
""" fast creation of a new array, given a template array.
type will be same as template.
if zero is true, new array will be initialized with zeroes."""
cdef array op
op = newarrayobject(template.ob_type, length, template.ob_descr)
if zero and op is not None:
memset(op._c, 0, length * op.ob_descr.itemsize)
return op

cdef inline array copy(array self):
cdef array op = newarrayobject(<PyTypeObject*>self.ob_type, self.ob_size,
self.ob_descr)
memcpy(op.ob_item, self.ob_item, op.ob_size * op.ob_descr.itemsize)
""" make a copy of an array. """
cdef array op
op = newarrayobject(self.ob_type, self.length, self.ob_descr)
memcpy(op._c, self._c, op.length * op.ob_descr.itemsize)
return op

cdef inline int extend_buffer(array self, char* stuff, Py_ssize_t n):
""" efficent appending of new stuff of same type (e.g. of same array type)
n: number of elements (not number of bytes!)
"""
cdef Py_ssize_t itemsize = self.ob_descr.itemsize, orgsize = self.ob_size
if -1 == resize_smart(self, orgsize + n):
""" efficent appending of new stuff of same type
(e.g. of same array type)
n: number of elements (not number of bytes!) """
cdef Py_ssize_t itemsize = self.ob_descr.itemsize
cdef Py_ssize_t orgsize = self.length
if resize_smart(self, orgsize + n) == -1:
return -1
memcpy(self.ob_item + orgsize * itemsize, stuff, n * itemsize)
memcpy(self._c + orgsize * itemsize, stuff, n * itemsize)

cdef inline int extend(array self, array other):
""" extend array with data from another array; types must match. """
if self.ob_descr.typecode != self.ob_descr.typecode:
PyErr_BadArgument()
return -1
return extend_buffer(self, other.ob_item, other.ob_size)

return extend_buffer(self, other._c, other.length)

cdef inline void zero(array op):
memset(op.ob_item, 0, op.ob_size * op.ob_descr.itemsize)
""" set all elements of array to zero. """
memset(op._c, 0, op.length * op.ob_descr.itemsize)
99 changes: 50 additions & 49 deletions Cython/Includes/cpython/arrayarray.h
View file
Expand Up @@ -6,6 +6,7 @@
See array.pxd next to this file
last changes: 2009-05-15 rk
2012-05-02 andreasvc
*/

Expand All @@ -21,24 +22,24 @@ struct arrayobject; /* Forward */
* functions aren't visible yet.
*/
typedef struct arraydescr {
int typecode;
int itemsize;
PyObject * (*getitem)(struct arrayobject *, Py_ssize_t);
int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *);
int typecode;
int itemsize;
PyObject * (*getitem)(struct arrayobject *, Py_ssize_t);
int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *);
#if PY_VERSION_HEX >= 0x03000000
char *formats;
#endif
} arraydescr;


typedef struct arrayobject {
PyObject_HEAD
PyObject_HEAD
union {
int ob_size;
Py_ssize_t ob_size;
unsigned length;
};
union {
char *ob_item;
char *ob_item;
float *_f;
double *_d;
int *_i;
Expand All @@ -54,11 +55,11 @@ typedef struct arrayobject {
void *_v;
};
#if PY_VERSION_HEX >= 0x02040000
Py_ssize_t allocated;
Py_ssize_t allocated;
#endif
struct arraydescr *ob_descr;
struct arraydescr *ob_descr;
#if PY_VERSION_HEX >= 0x02040000
PyObject *weakreflist; /* List of weak references */
PyObject *weakreflist; /* List of weak references */
#if PY_VERSION_HEX >= 0x03000000
int ob_exports; /* Number of exported buffers */
#endif
Expand All @@ -69,50 +70,50 @@ typedef struct arrayobject {
#ifndef NO_NEWARRAY_INLINE
/*
*
* fast creation of a new array - init with zeros
* fast creation of a new array
*/

PyObject * newarrayobject
(PyTypeObject *type, Py_ssize_t size, struct arraydescr *descr) {
arrayobject *op;
size_t nbytes;

if (size < 0) {
PyErr_BadInternalCall();
return NULL;
}

nbytes = size * descr->itemsize;
/* Check for overflow */
if (nbytes / descr->itemsize != (size_t)size) {
return PyErr_NoMemory();
}
op = (arrayobject *) type->tp_alloc(type, 0);
if (op == NULL) {
return NULL;
}
op->ob_descr = descr;
inline PyObject * newarrayobject(PyTypeObject *type, Py_ssize_t size,

This comment has been minimized.

Sign in to view

Copy link
@scoder

scoder May 6, 2012

Note that "inline" isn't portable. Either CYTHON_INLINE or Py_LOCAL_INLINE should be used here, but I'm not currently sure which is better.

This comment has been minimized.

Sign in to view

Copy link
@andreasvc

andreasvc May 6, 2012

Author Owner

It's not, but I don't know the details about the two you mention; e.g., the latter makes a function "static" as well, is that a problem? Either way, the way it is now one can #define NO_NEWARRAY_INLINE to disable the inlines, because they just redefine existing functions to optionally make them inline.

This comment has been minimized.

Sign in to view

Copy link
@scoder

scoder via email May 6, 2012

This comment has been minimized.

Sign in to view

Copy link
@andreasvc

andreasvc May 6, 2012

Author Owner

When I simply replace inline with either of those two, I get the following error:

arrayarray.h:75: error: expected ?=?, ?,?, ?;?, ?asm? or ?__attribute__? before ?PyObject?
_fragmentseeker.c: In function ?__pyx_f_5array_clone?:
_fragmentseeker.c:14551: warning: implicit declaration of function ?newarrayobject?
_fragmentseeker.c:14551: warning: cast to pointer from integer of different size
_fragmentseeker.c: In function ?__pyx_f_5array_copy?:
_fragmentseeker.c:14633: warning: cast to pointer from integer of different size
error: command 'gcc' failed with exit status 1
make: *** [all] Error 1

So the function is no longer accessible from other files.
struct arraydescr *descr) {
arrayobject *op;
size_t nbytes;

if (size < 0) {
PyErr_BadInternalCall();
return NULL;
}

nbytes = size * descr->itemsize;
/* Check for overflow */
if (nbytes / descr->itemsize != (size_t)size) {
return PyErr_NoMemory();
}
op = (arrayobject *) type->tp_alloc(type, 0);
if (op == NULL) {
return NULL;
}
op->ob_descr = descr;
#if !( PY_VERSION_HEX < 0x02040000 )
op->allocated = size;
op->weakreflist = NULL;
op->allocated = size;
op->weakreflist = NULL;
#endif
Py_SIZE(op) = size;
if (size <= 0) {
op->ob_item = NULL;
}
else {
op->ob_item = PyMem_NEW(char, nbytes);
if (op->ob_item == NULL) {
Py_DECREF(op);
return PyErr_NoMemory();
}
}
return (PyObject *) op;
Py_SIZE(op) = size;
if (size <= 0) {
op->ob_item = NULL;
}
else {
op->ob_item = PyMem_NEW(char, nbytes);
if (op->ob_item == NULL) {
Py_DECREF(op);
return PyErr_NoMemory();
}
}
return (PyObject *) op;
}
#else
PyObject *
newarrayobject(PyTypeObject *type, Py_ssize_t size, struct arraydescr *descr);
#endif
PyObject* newarrayobject(PyTypeObject *type, Py_ssize_t size,
struct arraydescr *descr);
#endif /* ifndef NO_NEWARRAY_INLINE */

/* fast resize (reallocation to the point)
not designed for filing small increments (but for fast opaque array apps) */
Expand Down Expand Up @@ -143,7 +144,7 @@ int resize_smart(arrayobject *self, Py_ssize_t n) {
return 0;
}
}
newsize = n * 3 / 2 + 1;
newsize = n * 3 / 2 + 1;
PyMem_Resize(item, char, (size_t)(newsize * self->ob_descr->itemsize));
if (item == NULL) {
PyErr_NoMemory();
Expand Down

3 comments on commit 13d9929

@andreasvc
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder whether getbuffer and releasebuffer are important. Right now they just generate warnings on every compile and I'm tempted to just remove them.

@scoder
Copy link

@scoder scoder commented on 13d9929 May 6, 2012 via email

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@andreasvc
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added the tests. I should say I haven't figured out how to run them. When I run the whole Cython test suite the array module doesn't get compiled correctly, but I wouldn't know how to debug it.

Please sign in to comment.