Skip to content

Commit

Permalink
TST,MAINT: New tests, byteswap cleanups and fixed assert
Browse files Browse the repository at this point in the history
  • Loading branch information
@seberg
seberg committed Jan 15, 2022
1 parent d2473c0 commit 245af22
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 26 deletions.
22 changes: 16 additions & 6 deletions numpy/core/src/multiarray/textreading/conversions.c
View file
Expand Up @@ -5,6 +5,10 @@
#include <stdlib.h>
#include <stdbool.h>

#define NPY_NO_DEPRECATED_API NPY_API_VERSION
#define _MULTIARRAYMODULE
#include "lowlevel_strided_loops.h"

#include "conversions.h"
#include "str_to_int.h"

Expand Down Expand Up @@ -122,7 +126,7 @@ to_float(PyArray_Descr *descr,
float val = (float)double_val;
memcpy(dataptr, &val, sizeof(float));
if (!PyArray_ISNBO(descr->byteorder)) {
descr->f->copyswap(dataptr, dataptr, 1, NULL);
npy_bswap4_unaligned(dataptr);
}
return 0;
}
Expand All @@ -144,7 +148,7 @@ to_double(PyArray_Descr *descr,

memcpy(dataptr, &val, sizeof(double));
if (!PyArray_ISNBO(descr->byteorder)) {
descr->f->copyswap(dataptr, dataptr, 1, NULL);
npy_bswap8_unaligned(dataptr);
}
return 0;
}
Expand Down Expand Up @@ -241,7 +245,8 @@ to_cfloat(PyArray_Descr *descr,
npy_complex64 val = {(float)real, (float)imag};
memcpy(dataptr, &val, sizeof(npy_complex64));
if (!PyArray_ISNBO(descr->byteorder)) {
descr->f->copyswap(dataptr, dataptr, 1, NULL);
npy_bswap4_unaligned(dataptr);
npy_bswap4_unaligned(dataptr + 4);
}
return 0;
}
Expand All @@ -264,7 +269,8 @@ to_cdouble(PyArray_Descr *descr,
npy_complex128 val = {real, imag};
memcpy(dataptr, &val, sizeof(npy_complex128));
if (!PyArray_ISNBO(descr->byteorder)) {
descr->f->copyswap(dataptr, dataptr, 1, NULL);
npy_bswap8_unaligned(dataptr);
npy_bswap8_unaligned(dataptr + 8);
}
return 0;
}
Expand Down Expand Up @@ -319,7 +325,11 @@ to_unicode(PyArray_Descr *descr,
}

if (!PyArray_ISNBO(descr->byteorder)) {
descr->f->copyswap(dataptr, dataptr, 1, NULL);
/* manual byteswap, unicode requires the array to be passed... */
for (int i = 0; i < descr->elsize; i++) {
npy_bswap4_unaligned(dataptr);
dataptr += 4;
}
}
return 0;
}
Expand Down Expand Up @@ -383,4 +393,4 @@ to_generic(PyArray_Descr *descr,
parser_config *config)
{
return to_generic_with_converter(descr, str, end, dataptr, config, NULL);
}
}
4 changes: 3 additions & 1 deletion numpy/core/src/multiarray/textreading/rows.c
View file
Expand Up @@ -176,6 +176,7 @@ read_rows(stream *s,
Py_XINCREF(data_array);
size_t rows_per_block = 1; /* will be increased depending on row size */
npy_intp data_allocated_rows = 0;

/* We give a warning if max_rows is used and an empty line is encountered */
bool give_empty_row_warning = max_rows >= 0;

Expand All @@ -188,10 +189,11 @@ read_rows(stream *s,
/* Set the actual number of fields if it is already known, otherwise -1 */
Py_ssize_t actual_num_fields = -1;
if (usecols != NULL) {
assert(homogeneous || num_field_types == num_usecols);
actual_num_fields = num_usecols;
assert(num_field_types == num_usecols);
}
else if (!homogeneous) {
assert(usecols == NULL || num_field_types == num_usecols);
actual_num_fields = num_field_types;
}

Expand Down
31 changes: 18 additions & 13 deletions numpy/core/src/multiarray/textreading/str_to_int.c
View file
@@ -1,13 +1,16 @@

#include <Python.h>

#define NPY_NO_DEPRECATED_API NPY_API_VERSION
#define _MULTIARRAYMODULE
#include "lowlevel_strided_loops.h"

#include <string.h>
#include "textreading/str_to_int.h"
#include "textreading/conversions.h"
#include "textreading/parser_config.h"


#define DECLARE_TO_INT(intw, INT_MIN, INT_MAX) \
#define DECLARE_TO_INT(intw, INT_MIN, INT_MAX, byteswap_unaligned) \
NPY_NO_EXPORT int \
to_##intw(PyArray_Descr *descr, \
const Py_UCS4 *str, const Py_UCS4 *end, char *dataptr, \
Expand All @@ -24,12 +27,12 @@
} \
memcpy(dataptr, &x, sizeof(x)); \
if (!PyArray_ISNBO(descr->byteorder)) { \
descr->f->copyswap(dataptr, dataptr, 1, NULL); \
byteswap_unaligned(dataptr); \
} \
return 0; \
}

#define DECLARE_TO_UINT(uintw, UINT_MAX) \
#define DECLARE_TO_UINT(uintw, UINT_MAX, byteswap_unaligned) \
NPY_NO_EXPORT int \
to_##uintw(PyArray_Descr *descr, \
const Py_UCS4 *str, const Py_UCS4 *end, char *dataptr, \
Expand All @@ -46,17 +49,19 @@
} \
memcpy(dataptr, &x, sizeof(x)); \
if (!PyArray_ISNBO(descr->byteorder)) { \
descr->f->copyswap(dataptr, dataptr, 1, NULL); \
byteswap_unaligned(dataptr); \
} \
return 0; \
}

DECLARE_TO_INT(int8, INT8_MIN, INT8_MAX)
DECLARE_TO_INT(int16, INT16_MIN, INT16_MAX)
DECLARE_TO_INT(int32, INT32_MIN, INT32_MAX)
DECLARE_TO_INT(int64, INT64_MIN, INT64_MAX)
#define byteswap_nothing(ptr)

DECLARE_TO_INT(int8, INT8_MIN, INT8_MAX, byteswap_nothing)
DECLARE_TO_INT(int16, INT16_MIN, INT16_MAX, npy_bswap2_unaligned)
DECLARE_TO_INT(int32, INT32_MIN, INT32_MAX, npy_bswap4_unaligned)
DECLARE_TO_INT(int64, INT64_MIN, INT64_MAX, npy_bswap8_unaligned)

DECLARE_TO_UINT(uint8, UINT8_MAX)
DECLARE_TO_UINT(uint16, UINT16_MAX)
DECLARE_TO_UINT(uint32, UINT32_MAX)
DECLARE_TO_UINT(uint64, UINT64_MAX)
DECLARE_TO_UINT(uint8, UINT8_MAX, byteswap_nothing)
DECLARE_TO_UINT(uint16, UINT16_MAX, npy_bswap2_unaligned)
DECLARE_TO_UINT(uint32, UINT32_MAX, npy_bswap4_unaligned)
DECLARE_TO_UINT(uint64, UINT64_MAX, npy_bswap8_unaligned)
31 changes: 25 additions & 6 deletions numpy/lib/tests/test_io.py
View file
Expand Up @@ -3288,18 +3288,27 @@ def test_loadtxt_consecutive_quotechar_escaped():

@pytest.mark.parametrize("data", ("", "\n\n\n", "# 1 2 3\n# 4 5 6\n"))
@pytest.mark.parametrize("ndmin", (0, 1, 2))
def test_loadtxt_warn_on_no_data(data, ndmin):
@pytest.mark.parametrize("usecols", [None, (1, 2, 3)])
def test_loadtxt_warn_on_no_data(data, ndmin, usecols):
"""Check that a UserWarning is emitted when no data is read from input."""
if usecols is not None:
expected_shape = (0, 3)
elif ndmin == 2:
expected_shape = (0, 1) # guess a single column?!
else:
expected_shape = (0,)

txt = TextIO(data)
with pytest.warns(UserWarning, match="input contained no data"):
np.loadtxt(txt, ndmin=ndmin)
res = np.loadtxt(txt, ndmin=ndmin, usecols=usecols)
assert res.shape == expected_shape

with NamedTemporaryFile(mode="w") as fh:
fh.write(data)
fh.seek(0)
with pytest.warns(UserWarning, match="input contained no data"):
np.loadtxt(txt, ndmin=ndmin)

res = np.loadtxt(txt, ndmin=ndmin, usecols=usecols)
assert res.shape == expected_shape

@pytest.mark.parametrize("skiprows", (2, 3))
def test_loadtxt_warn_on_skipped_data(skiprows):
Expand All @@ -3309,7 +3318,7 @@ def test_loadtxt_warn_on_skipped_data(skiprows):
np.loadtxt(txt, skiprows=skiprows)

@pytest.mark.parametrize("dtype",
np.typecodes["AllInteger"] + np.typecodes["AllFloat"])
list(np.typecodes["AllInteger"] + np.typecodes["AllFloat"]) + ["U2"])
@pytest.mark.parametrize("swap", [True, False])
def test_loadtxt_byteswapping_and_unaligned(dtype, swap):
data = ["x,1\n"] # no need for complicated data
Expand All @@ -3320,7 +3329,7 @@ def test_loadtxt_byteswapping_and_unaligned(dtype, swap):
# The above ensures that the interesting "b" field is unaligned:
assert full_dt.fields["b"][1] == 1
res = np.loadtxt(data, dtype=full_dt, delimiter=",")
assert res["b"] == 1
assert res["b"] == dtype.type(1)

@pytest.mark.parametrize("dtype",
np.typecodes["AllInteger"] + "efdFD" + "?")
Expand Down Expand Up @@ -3406,3 +3415,13 @@ def test_not_an_iter(self):
match="error reading from object, expected an iterable"):
np.core._multiarray_umath._load_from_filelike(
object(), dtype=np.dtype("i"), filelike=False)

def test_bad_type(self):
with pytest.raises(TypeError, match="internal error: dtype must"):
np.core._multiarray_umath._load_from_filelike(
object(), dtype="i", filelike=False)

def test_bad_encoding(self):
with pytest.raises(TypeError, match="encoding must be a unicode"):
np.core._multiarray_umath._load_from_filelike(
object(), dtype=np.dtype("i"), filelike=False, encoding=123)

0 comments on commit 245af22

Please sign in to comment.