Awkward conversion tests are failing with numpy 1.25

[eriknw]

As seen here (https://github.com/python-graphblas/python-graphblas/actions/runs/5337195569/jobs/9672901538#step:9:731), the tests for awkward are failing. I don't know why or when or for what environments they are failing.

___________________________ test_awkward_roundtrip ____________________________

array = <Array [{data: [...], ...}, {...}, {...}] type='3 * struct[{data: var * {in...'>
name = '_AwkwardDoublyCompressedMatrix'

    def with_name(array, name, *, highlevel=True, behavior=None):
        """
        Args:
            array: Array-like data (anything #ak.to_layout recognizes).
            name (str or None): Name to give to the records or tuples; this assigns
                the `"__record__"` parameter. If None, any existing name is unset.
            highlevel (bool): If True, return an #ak.Array; otherwise, return
                a low-level #ak.contents.Content subclass.
            behavior (None or dict): Custom #ak.behavior for the output array, if
                high-level.
    
        Returns an #ak.Array or #ak.Record (or low-level equivalent, if
        `highlevel=False`) with a new name. This function does not change the
        array in-place. If the new name is None, then an array without a name is
        returned.
    
        The records or tuples may be nested within multiple levels of nested lists.
        If records are nested within records, only the outermost are affected.
    
        Setting the `"__record__"` parameter makes it possible to add behaviors
        to the data; see #ak.Array and #ak.behavior for a more complete
        description.
        """
        with ak._errors.OperationErrorContext(
            "ak.with_name",
            {"array": array, "name": name, "highlevel": highlevel, "behavior": behavior},
        ):
>           return _impl(array, name, highlevel, behavior)

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\operations\ak_with_name.py:38: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

array = <Array [{data: [...], ...}, {...}, {...}] type='3 * struct[{data: var * {in...'>
name = '_AwkwardDoublyCompressedMatrix', highlevel = True, behavior = None

    def _impl(array, name, highlevel, behavior):
        behavior = behavior_of(array, behavior=behavior)
        layout = ak.operations.to_layout(array)
    
        def action(layout, **ignore):
            if isinstance(layout, ak.contents.RecordArray):
                return ak.contents.RecordArray(
                    layout._contents,
                    layout._fields,
                    layout._length,
                    parameters={**layout.parameters, "__record__": name},
                )
            else:
                return None
    
        out = ak._do.recursively_apply(layout, action, behavior)
    
        def action2(layout, **ignore):
            if layout.is_union:
                return ak.contents.UnionArray.simplified(
                    layout._tags,
                    layout._index,
                    layout._contents,
                    parameters=layout._parameters,
                )
            else:
                return None
    
        out2 = ak._do.recursively_apply(out, action2, behavior)
    
>       return wrap_layout(out2, behavior, highlevel)

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\operations\ak_with_name.py:71: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

content = <RecordArray is_tuple='false' len='3'>
    <parameter name='format'>'hypercsr'</parameter>
    <parameter name='shape'...'1' field='offset_labels'>
        <NumpyArray dtype='int64' len='3'>[0 3 5]</NumpyArray>
    </content>
</RecordArray>
behavior = None, highlevel = True, like = None, allow_other = False

    def wrap_layout(content, behavior=None, highlevel=True, like=None, allow_other=False):
        import awkward.highlevel
        from awkward.contents import Content
        from awkward.record import Record
    
        assert content is None or isinstance(content, (Content, Record)) or allow_other
        assert behavior is None or isinstance(behavior, Mapping)
        assert isinstance(highlevel, bool)
        if highlevel:
            if like is not None and behavior is None:
                behavior = behavior_of(like)
    
            if isinstance(content, Content):
>               return awkward.highlevel.Array(content, behavior=behavior)

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\_layout.py:32: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <[AttributeError("no field named '_behavior'") raised in repr()] Array object at 0x2e8f2b1ab30>
data = <RecordArray is_tuple='false' len='3'>
    <parameter name='format'>'hypercsr'</parameter>
    <parameter name='shape'...'1' field='offset_labels'>
        <NumpyArray dtype='int64' len='3'>[0 3 5]</NumpyArray>
    </content>
</RecordArray>

    def __init__(
        self,
        data,
        *,
        behavior=None,
        with_name=None,
        check_valid=False,
        backend=None,
    ):
        self._cpp_type = None
        if isinstance(data, ak.contents.Content):
            layout = data
    
        elif isinstance(data, Array):
            layout = data._layout
            behavior = behavior_of(data, behavior=behavior)
    
        elif isinstance(data, dict):
            fields = []
            contents = []
            length = None
            for k, v in data.items():
                fields.append(k)
                contents.append(Array(v).layout)
                if length is None:
                    length = contents[-1].length
                elif length != contents[-1].length:
                    raise ValueError(
                        "dict of arrays in ak.Array constructor must have arrays "
                        "of equal length ({} vs {})".format(length, contents[-1].length)
                    )
            layout = ak.contents.RecordArray(contents, fields)
    
        elif isinstance(data, str):
            layout = ak.operations.from_json(data, highlevel=False)
    
        else:
            layout = ak.operations.to_layout(
                data, allow_record=False, regulararray=False
            )
    
        if not isinstance(layout, ak.contents.Content):
            raise TypeError("could not convert data into an ak.Array")
    
        if with_name is not None:
            layout = ak.operations.with_name(
                layout, with_name, highlevel=False, behavior=behavior
            )
    
        if backend is not None and backend != ak.operations.backend(layout):
            layout = ak.operations.to_backend(layout, backend, highlevel=False)
    
        self.layout = layout
>       self.behavior = behavior

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\highlevel.py:230: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <[AttributeError("no field named '_behavior'") raised in repr()] Array object at 0x2e8f2b1ab30>
name = 'behavior', value = None

    def __setattr__(self, name, value):
        """
        Args:
            where (str): Attribute name to set
    
        Set an attribute on the array.
    
        Only existing public attributes e.g. #ak.Array.layout, or private
        attributes (with leading underscores), can be set.
    
        Fields are not assignable to as attributes, i.e. the following doesn't work:
    
            array.z = new_field
    
        Instead, always use #ak.Array.__setitem__:
    
            array["z"] = new_field
    
        or #ak.with_field:
    
            array = ak.with_field(array, new_field, "z")
    
        to add or modify a field.
        """
        if name.startswith("_") or hasattr(type(self), name):
>           super().__setattr__(name, value)

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\highlevel.py:1140: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <[AttributeError("no field named '_behavior'") raised in repr()] Array object at 0x2e8f2b1ab30>
behavior = None

    @behavior.setter
    def behavior(self, behavior):
        if behavior is None or isinstance(behavior, Mapping):
>           self.__class__ = get_array_class(self._layout, behavior)

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\highlevel.py:319: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <[AttributeError("no field named '_behavior'") raised in repr()] Array object at 0x2e8f2b1ab30>
name = '__class__'
value = <class 'graphblas.io._awkward._AwkwardDoublyCompressedMatrixArray'>

    def __setattr__(self, name, value):
        """
        Args:
            where (str): Attribute name to set
    
        Set an attribute on the array.
    
        Only existing public attributes e.g. #ak.Array.layout, or private
        attributes (with leading underscores), can be set.
    
        Fields are not assignable to as attributes, i.e. the following doesn't work:
    
            array.z = new_field
    
        Instead, always use #ak.Array.__setitem__:
    
            array["z"] = new_field
    
        or #ak.with_field:
    
            array = ak.with_field(array, new_field, "z")
    
        to add or modify a field.
        """
        if name.startswith("_") or hasattr(type(self), name):
>           super().__setattr__(name, value)
E           TypeError: __class__ assignment: '_AwkwardDoublyCompressedMatrixArray' object layout differs from 'Array'

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\highlevel.py:1140: TypeError

The above exception was the direct cause of the following exception:

    @pytest.mark.skipif("not ak")
    def test_awkward_roundtrip():
        # Vector
        v = gb.Vector.from_coo([1, 3, 5], [20, 21, -5], size=22)
        for dtype in ["int16", "float32", "bool"]:
            v1 = v.dup(dtype=dtype)
            kv = gb.io.to_awkward(v1)
            assert isinstance(kv, ak.Array)
            v2 = gb.io.from_awkward(kv)
            assert v2.isequal(v1)
        # Matrix
        m = gb.Matrix.from_coo([0, 0, 3, 5], [1, 4, 0, 2], [1, 0, 2, -1], nrows=7, ncols=6)
        for dtype in ["int16", "float32", "bool"]:
            for format in ["csr", "csc", "hypercsr", "hypercsc"]:
                m1 = m.dup(dtype=dtype)
>               km = gb.io.to_awkward(m1, format=format)

graphblas\tests\test_io.py:386: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
graphblas\io\_awkward.py:180: in to_awkward
    ret = ak.with_name(ret, classname)
C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\operations\ak_with_name.py:34: in with_name
    with ak._errors.OperationErrorContext(
C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\_errors.py:56: in __exit__
    self.handle_exception(exception_type, exception_value)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <awkward._errors.OperationErrorContext object at 0x000002E8F2B189D0>
cls = <class 'TypeError'>
exception = TypeError("__class__ assignment: '_AwkwardDoublyCompressedMatrixArray' object layout differs from 'Array'")

    def handle_exception(self, cls: type[E], exception: E) -> E:
        if sys.version_info >= (3, 11, 0, "final"):
            self.decorate_exception(cls, exception)
        else:
>           raise self.decorate_exception(cls, exception)
E           TypeError: __class__ assignment: '_AwkwardDoublyCompressedMatrixArray' object layout differs from 'Array'
E           
E           This error occurred while calling
E           
E               ak.with_name(
E                   array = <Array [{data: [...], ...}, ..., {...}] type='3 * struct[{d...'>
E                   name = '_AwkwardDoublyCompressedMatrix'
E                   highlevel = True
E                   behavior = None
E               )

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\_errors.py:71: TypeError

_________________________ test_awkward_iso_roundtrip __________________________

array = <Array [{data: [...], ...}, {...}, {...}] type='3 * struct[{data: var * {in...'>
name = '_AwkwardDoublyCompressedMatrix'

    def with_name(array, name, *, highlevel=True, behavior=None):
        """
        Args:
            array: Array-like data (anything #ak.to_layout recognizes).
            name (str or None): Name to give to the records or tuples; this assigns
                the `"__record__"` parameter. If None, any existing name is unset.
            highlevel (bool): If True, return an #ak.Array; otherwise, return
                a low-level #ak.contents.Content subclass.
            behavior (None or dict): Custom #ak.behavior for the output array, if
                high-level.
    
        Returns an #ak.Array or #ak.Record (or low-level equivalent, if
        `highlevel=False`) with a new name. This function does not change the
        array in-place. If the new name is None, then an array without a name is
        returned.
    
        The records or tuples may be nested within multiple levels of nested lists.
        If records are nested within records, only the outermost are affected.
    
        Setting the `"__record__"` parameter makes it possible to add behaviors
        to the data; see #ak.Array and #ak.behavior for a more complete
        description.
        """
        with ak._errors.OperationErrorContext(
            "ak.with_name",
            {"array": array, "name": name, "highlevel": highlevel, "behavior": behavior},
        ):
>           return _impl(array, name, highlevel, behavior)

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\operations\ak_with_name.py:38: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

array = <Array [{data: [...], ...}, {...}, {...}] type='3 * struct[{data: var * {in...'>
name = '_AwkwardDoublyCompressedMatrix', highlevel = True, behavior = None

    def _impl(array, name, highlevel, behavior):
        behavior = behavior_of(array, behavior=behavior)
        layout = ak.operations.to_layout(array)
    
        def action(layout, **ignore):
            if isinstance(layout, ak.contents.RecordArray):
                return ak.contents.RecordArray(
                    layout._contents,
                    layout._fields,
                    layout._length,
                    parameters={**layout.parameters, "__record__": name},
                )
            else:
                return None
    
        out = ak._do.recursively_apply(layout, action, behavior)
    
        def action2(layout, **ignore):
            if layout.is_union:
                return ak.contents.UnionArray.simplified(
                    layout._tags,
                    layout._index,
                    layout._contents,
                    parameters=layout._parameters,
                )
            else:
                return None
    
        out2 = ak._do.recursively_apply(out, action2, behavior)
    
>       return wrap_layout(out2, behavior, highlevel)

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\operations\ak_with_name.py:71: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

content = <RecordArray is_tuple='false' len='3'>
    <parameter name='format'>'hypercsr'</parameter>
    <parameter name='shape'...'1' field='offset_labels'>
        <NumpyArray dtype='int64' len='3'>[0 3 5]</NumpyArray>
    </content>
</RecordArray>
behavior = None, highlevel = True, like = None, allow_other = False

    def wrap_layout(content, behavior=None, highlevel=True, like=None, allow_other=False):
        import awkward.highlevel
        from awkward.contents import Content
        from awkward.record import Record
    
        assert content is None or isinstance(content, (Content, Record)) or allow_other
        assert behavior is None or isinstance(behavior, Mapping)
        assert isinstance(highlevel, bool)
        if highlevel:
            if like is not None and behavior is None:
                behavior = behavior_of(like)
    
            if isinstance(content, Content):
>               return awkward.highlevel.Array(content, behavior=behavior)

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\_layout.py:32: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <[AttributeError("no field named '_behavior'") raised in repr()] Array object at 0x2e8f2b1b640>
data = <RecordArray is_tuple='false' len='3'>
    <parameter name='format'>'hypercsr'</parameter>
    <parameter name='shape'...'1' field='offset_labels'>
        <NumpyArray dtype='int64' len='3'>[0 3 5]</NumpyArray>
    </content>
</RecordArray>

    def __init__(
        self,
        data,
        *,
        behavior=None,
        with_name=None,
        check_valid=False,
        backend=None,
    ):
        self._cpp_type = None
        if isinstance(data, ak.contents.Content):
            layout = data
    
        elif isinstance(data, Array):
            layout = data._layout
            behavior = behavior_of(data, behavior=behavior)
    
        elif isinstance(data, dict):
            fields = []
            contents = []
            length = None
            for k, v in data.items():
                fields.append(k)
                contents.append(Array(v).layout)
                if length is None:
                    length = contents[-1].length
                elif length != contents[-1].length:
                    raise ValueError(
                        "dict of arrays in ak.Array constructor must have arrays "
                        "of equal length ({} vs {})".format(length, contents[-1].length)
                    )
            layout = ak.contents.RecordArray(contents, fields)
    
        elif isinstance(data, str):
            layout = ak.operations.from_json(data, highlevel=False)
    
        else:
            layout = ak.operations.to_layout(
                data, allow_record=False, regulararray=False
            )
    
        if not isinstance(layout, ak.contents.Content):
            raise TypeError("could not convert data into an ak.Array")
    
        if with_name is not None:
            layout = ak.operations.with_name(
                layout, with_name, highlevel=False, behavior=behavior
            )
    
        if backend is not None and backend != ak.operations.backend(layout):
            layout = ak.operations.to_backend(layout, backend, highlevel=False)
    
        self.layout = layout
>       self.behavior = behavior

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\highlevel.py:230: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <[AttributeError("no field named '_behavior'") raised in repr()] Array object at 0x2e8f2b1b640>
name = 'behavior', value = None

    def __setattr__(self, name, value):
        """
        Args:
            where (str): Attribute name to set
    
        Set an attribute on the array.
    
        Only existing public attributes e.g. #ak.Array.layout, or private
        attributes (with leading underscores), can be set.
    
        Fields are not assignable to as attributes, i.e. the following doesn't work:
    
            array.z = new_field
    
        Instead, always use #ak.Array.__setitem__:
    
            array["z"] = new_field
    
        or #ak.with_field:
    
            array = ak.with_field(array, new_field, "z")
    
        to add or modify a field.
        """
        if name.startswith("_") or hasattr(type(self), name):
>           super().__setattr__(name, value)

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\highlevel.py:1140: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <[AttributeError("no field named '_behavior'") raised in repr()] Array object at 0x2e8f2b1b640>
behavior = None

    @behavior.setter
    def behavior(self, behavior):
        if behavior is None or isinstance(behavior, Mapping):
>           self.__class__ = get_array_class(self._layout, behavior)

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\highlevel.py:319: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <[AttributeError("no field named '_behavior'") raised in repr()] Array object at 0x2e8f2b1b640>
name = '__class__'
value = <class 'graphblas.io._awkward._AwkwardDoublyCompressedMatrixArray'>

    def __setattr__(self, name, value):
        """
        Args:
            where (str): Attribute name to set
    
        Set an attribute on the array.
    
        Only existing public attributes e.g. #ak.Array.layout, or private
        attributes (with leading underscores), can be set.
    
        Fields are not assignable to as attributes, i.e. the following doesn't work:
    
            array.z = new_field
    
        Instead, always use #ak.Array.__setitem__:
    
            array["z"] = new_field
    
        or #ak.with_field:
    
            array = ak.with_field(array, new_field, "z")
    
        to add or modify a field.
        """
        if name.startswith("_") or hasattr(type(self), name):
>           super().__setattr__(name, value)
E           TypeError: __class__ assignment: '_AwkwardDoublyCompressedMatrixArray' object layout differs from 'Array'

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\highlevel.py:1140: TypeError

The above exception was the direct cause of the following exception:

    @pytest.mark.skipif("not ak")
    def test_awkward_iso_roundtrip():
        # Vector
        v = gb.Vector.from_coo([1, 3, 5], [20, 20, 20], size=22)
        if suitesparse:
            assert v.ss.is_iso
        kv = gb.io.to_awkward(v)
        assert isinstance(kv, ak.Array)
        v2 = gb.io.from_awkward(kv)
        assert v2.isequal(v)
        # Matrix
        m = gb.Matrix.from_coo([0, 0, 3, 5], [1, 4, 0, 2], [1, 1, 1, 1], nrows=7, ncols=6)
        if suitesparse:
            assert m.ss.is_iso
        for format in ["csr", "csc", "hypercsr", "hypercsc"]:
>           km = gb.io.to_awkward(m, format=format)

graphblas\tests\test_io.py:407: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
graphblas\io\_awkward.py:180: in to_awkward
    ret = ak.with_name(ret, classname)
C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\operations\ak_with_name.py:34: in with_name
    with ak._errors.OperationErrorContext(
C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\_errors.py:56: in __exit__
    self.handle_exception(exception_type, exception_value)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <awkward._errors.OperationErrorContext object at 0x000002E8F2B1A860>
cls = <class 'TypeError'>
exception = TypeError("__class__ assignment: '_AwkwardDoublyCompressedMatrixArray' object layout differs from 'Array'")

    def handle_exception(self, cls: type[E], exception: E) -> E:
        if sys.version_info >= (3, 11, 0, "final"):
            self.decorate_exception(cls, exception)
        else:
>           raise self.decorate_exception(cls, exception)
E           TypeError: __class__ assignment: '_AwkwardDoublyCompressedMatrixArray' object layout differs from 'Array'
E           
E           This error occurred while calling
E           
E               ak.with_name(
E                   array = <Array [{data: [...], ...}, ..., {...}] type='3 * struct[{d...'>
E                   name = '_AwkwardDoublyCompressedMatrix'
E                   highlevel = True
E                   behavior = None
E               )

C:\Miniconda3\envs\graphblas\lib\site-packages\awkward\_errors.py:71: TypeError

For now, I marked the tests as xfail until we can understand and fix them.

[SultanOrazbayev]

Would it be easier to debug if we restore the full matrix of testing (not skipping versions)? Also, I hope the order of the functions (which I changed in the io for awkward recently) doesn't cause this... it seems to be fine for some of the tests, so probably not.

[eriknw]

Re-opened. #467 isolated the problem (numpy 1.25), but it is still xfailing. My plan is to hope awkward fixes the problem, but it would be best if somebody reported upstream.