tests fail on big endian platform (s390x)

[xnox]

Hello,

In Ubuntu we execute test-suite for asdf across all architectures. It passes on most architectures, but fails on s390x. It is are only big endian platform.

=================================== FAILURES ===================================
__________________________________ test_table __________________________________

tmpdir = local('/tmp/pytest-of-ubuntu/pytest-0/test_table0')

    def test_table(tmpdir):
        table = np.array(
            [(0, 1, (2, 3)), (4, 5, (6, 7))],
            dtype=[(str('MINE'), np.int8),
                   (str(''), np.float64),
                   (str('arr'), '>i4', (2,))])
    
        tree = {'table_data': table}
    
        def check_raw_yaml(content):
            tree = yaml.load(
                re.sub(br'!core/\S+', b'', content))
    
            assert tree['table_data'] == {
                'datatype': [
                    {'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'},
                    {'byteorder': 'little', 'datatype': 'float64', 'name': 'f1'},
                    {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}
                    ],
                'shape': [2],
                'source': 0,
                'byteorder': 'big'
                }
    
>       helpers.assert_roundtrip_tree(tree, tmpdir, None, check_raw_yaml)

/usr/lib/python2.7/dist-packages/asdf/tags/core/tests/test_ndarray.py:223: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib/python2.7/dist-packages/asdf/tests/helpers.py:131: in assert_roundtrip_tree
    raw_yaml_check_func(content)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

content = '%YAML 1.1\n%TAG ! tag:stsci.edu:asdf/\n--- !core/asdf-1.0.0\nasdf_library: !core/software-1.0.0 {author: Space Telesc...me: f1}\n  - byteorder: big\n    datatype: int32\n    name: arr\n    shape: [2]\n  byteorder: big\n  shape: [2]\n...\n'

    def check_raw_yaml(content):
        tree = yaml.load(
            re.sub(br'!core/\S+', b'', content))
    
>       assert tree['table_data'] == {
            'datatype': [
                {'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'},
                {'byteorder': 'little', 'datatype': 'float64', 'name': 'f1'},
                {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}
                ],
            'shape': [2],
            'source': 0,
            'byteorder': 'big'
            }
E       assert {'byteorder':..., 'source': 0} == {'byteorder': ..., 'source': 0}
E         Omitting 3 identical items, use -v to show
E         Differing items:
E         {'datatype': [{'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'}, {'byteorder': 'big', 'datatype': 'float64', 'name': 'f1'}, {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}]} != {'datatype': [{'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'}, {'byteorder': 'little', 'datatype': 'float64', 'name': 'f1'}, {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}]}
E         Use -v to get the full diff

/usr/lib/python2.7/dist-packages/asdf/tags/core/tests/test_ndarray.py:212: AssertionError
___________________________ test_table_nested_fields ___________________________

tmpdir = local('/tmp/pytest-of-ubuntu/pytest-0/test_table_nested_fields0')

    def test_table_nested_fields(tmpdir):
        table = np.array(
            [(0, (1, 2)), (4, (5, 6)), (7, (8, 9))],
            dtype=[(str('A'), np.int64),
                   (str('B'), [(str('C'), np.int64), (str('D'), np.int64)])])
    
        tree = {'table_data': table}
    
        def check_raw_yaml(content):
            tree = yaml.load(
                re.sub(br'!core/\S+', b'', content))
    
            assert tree['table_data'] == {
                'datatype': [
                    {'datatype': 'int64', 'name': 'A', 'byteorder': 'little'},
                    {'datatype': [
                        {'datatype': 'int64', 'name': 'C', 'byteorder': 'little'},
                        {'datatype': 'int64', 'name': 'D', 'byteorder': 'little'}
                    ], 'name': 'B', 'byteorder': 'big'}],
                'shape': [3],
                'source': 0,
                'byteorder': 'big'
            }
    
>       helpers.assert_roundtrip_tree(tree, tmpdir, None, check_raw_yaml)

/usr/lib/python2.7/dist-packages/asdf/tags/core/tests/test_ndarray.py:250: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib/python2.7/dist-packages/asdf/tests/helpers.py:131: in assert_roundtrip_tree
    raw_yaml_check_func(content)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

content = '%YAML 1.1\n%TAG ! tag:stsci.edu:asdf/\n--- !core/asdf-1.0.0\nasdf_library: !core/software-1.0.0 {author: Space Telesc... int64, name: C}\n    - {byteorder: big, datatype: int64, name: D}\n    name: B\n  byteorder: big\n  shape: [3]\n...\n'

    def check_raw_yaml(content):
        tree = yaml.load(
            re.sub(br'!core/\S+', b'', content))
    
>       assert tree['table_data'] == {
            'datatype': [
                {'datatype': 'int64', 'name': 'A', 'byteorder': 'little'},
                {'datatype': [
                    {'datatype': 'int64', 'name': 'C', 'byteorder': 'little'},
                    {'datatype': 'int64', 'name': 'D', 'byteorder': 'little'}
                ], 'name': 'B', 'byteorder': 'big'}],
            'shape': [3],
            'source': 0,
            'byteorder': 'big'
        }
E       assert {'byteorder':..., 'source': 0} == {'byteorder': ..., 'source': 0}
E         Omitting 3 identical items, use -v to show
E         Differing items:
E         {'datatype': [{'byteorder': 'big', 'datatype': 'int64', 'name': 'A'}, {'byteorder': 'big', 'datatype': [{'byteorder': 'big', 'datatype': 'int64', 'name': 'C'}, {'byteorder': 'big', 'datatype': 'int64', 'name': 'D'}], 'name': 'B'}]} != {'datatype': [{'byteorder': 'little', 'datatype': 'int64', 'name': 'A'}, {'byteorder': 'big', 'datatype': [{'byteorder': 'little', 'datatype': 'int64', 'name': 'C'}, {'byteorder': 'little', 'datatype': 'int64', 'name': 'D'}], 'name': 'B'}]}
E         Use -v to get the full diff

/usr/lib/python2.7/dist-packages/asdf/tags/core/tests/test_ndarray.py:238: AssertionError
________________________________ test_checksum _________________________________

tmpdir = '/tmp/pytest-of-ubuntu/pytest-0/test_checksum0'

    def test_checksum(tmpdir):
        tmpdir = str(tmpdir)
        path = os.path.join(tmpdir, 'test.asdf')
    
        my_array = np.arange(0, 64, dtype=np.int64).reshape((8, 8))
        tree = {'my_array': my_array}
        ff = asdf.AsdfFile(tree)
        ff.write_to(path)
    
        with asdf.AsdfFile.open(path, validate_checksums=True) as ff:
            assert type(ff.blocks._internal_blocks[0].checksum) == bytes
>           assert ff.blocks._internal_blocks[0].checksum == \
                b'\xcaM\\\xb8t_L|\x00\n+\x01\xf1\xcfP1'
E           assert '\xab\xc17\xb...x97\x1b@\x86G' == '\xcaM\\\xb8t_...x01\xf1\xcfP1'
E             - \xab\xc17\xb9\xd9\x94~Z \xf3\x05\x97\x1b@\x86G
E             + \xcaM\\\xb8t_L|\x00
E             + +\x01\xf1\xcfP1

/usr/lib/python2.7/dist-packages/asdf/tests/test_low_level.py:740: AssertionError
_____________________________ test_checksum_update _____________________________

tmpdir = '/tmp/pytest-of-ubuntu/pytest-0/test_checksum_update0'

    def test_checksum_update(tmpdir):
        tmpdir = str(tmpdir)
        path = os.path.join(tmpdir, 'test.asdf')
    
        my_array = np.arange(0, 64, dtype=np.int64).reshape((8, 8))
    
        tree = {'my_array': my_array}
        ff = asdf.AsdfFile(tree)
        ff.write_to(path)
    
        with asdf.AsdfFile.open(path, mode='rw') as ff:
            ff.tree['my_array'][7, 7] = 0.0
            # update() should update the checksum, even if the data itself
            # is memmapped and isn't expressly re-written.
            ff.update()
    
        with asdf.AsdfFile.open(path, validate_checksums=True) as ff:
>           assert ff.blocks._internal_blocks[0].checksum == \
                b'T\xaf~[\x90\x8a\x88^\xc2B\x96D,N\xadL'
E           assert '\x01\xf5\xb2...x99`\xd7\xa9w' == 'T\xaf~[\x90\x...B\x96D,N\xadL'
E             - \x01\xf5\xb2<]\x9eQ\xfd\x02\xf9\x15\x99`\xd7\xa9w
E             + T\xaf~[\x90\x8a\x88^\xc2B\x96D,N\xadL

/usr/lib/python2.7/dist-packages/asdf/tests/test_low_level.py:761: AssertionError

And python3 too

=================================== FAILURES ===================================
__________________________________ test_table __________________________________

tmpdir = local('/tmp/pytest-of-ubuntu/pytest-0/test_table0')

    def test_table(tmpdir):
        table = np.array(
            [(0, 1, (2, 3)), (4, 5, (6, 7))],
            dtype=[(str('MINE'), np.int8),
                   (str(''), np.float64),
                   (str('arr'), '>i4', (2,))])
    
        tree = {'table_data': table}
    
        def check_raw_yaml(content):
            tree = yaml.load(
                re.sub(br'!core/\S+', b'', content))
    
            assert tree['table_data'] == {
                'datatype': [
                    {'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'},
                    {'byteorder': 'little', 'datatype': 'float64', 'name': 'f1'},
                    {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}
                    ],
                'shape': [2],
                'source': 0,
                'byteorder': 'big'
                }
    
>       helpers.assert_roundtrip_tree(tree, tmpdir, None, check_raw_yaml)

/usr/lib/python3/dist-packages/asdf/tags/core/tests/test_ndarray.py:223: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib/python3/dist-packages/asdf/tests/helpers.py:131: in assert_roundtrip_tree
    raw_yaml_check_func(content)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

content = b"%YAML 1.1\n%TAG ! tag:stsci.edu:asdf/\n--- !core/asdf-1.0.0\nasdf_library: !core/software-1.0.0 {author: Space Teles...me: f1}\n  - byteorder: big\n    datatype: int32\n    name: arr\n    shape: [2]\n  byteorder: big\n  shape: [2]\n...\n"

    def check_raw_yaml(content):
        tree = yaml.load(
            re.sub(br'!core/\S+', b'', content))
    
>       assert tree['table_data'] == {
            'datatype': [
                {'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'},
                {'byteorder': 'little', 'datatype': 'float64', 'name': 'f1'},
                {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}
                ],
            'shape': [2],
            'source': 0,
            'byteorder': 'big'
            }
E       assert {'byteorder':..., 'source': 0} == {'byteorder': ..., 'source': 0}
E         Omitting 3 identical items, use -v to show
E         Differing items:
E         {'datatype': [{'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'}, {'byteorder': 'big', 'datatype': 'float64', 'name': 'f1'}, {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}]} != {'datatype': [{'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'}, {'byteorder': 'little', 'datatype': 'float64', 'name': 'f1'}, {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}]}
E         Use -v to get the full diff

/usr/lib/python3/dist-packages/asdf/tags/core/tests/test_ndarray.py:212: AssertionError
___________________________ test_table_nested_fields ___________________________

tmpdir = local('/tmp/pytest-of-ubuntu/pytest-0/test_table_nested_fields0')

    def test_table_nested_fields(tmpdir):
        table = np.array(
            [(0, (1, 2)), (4, (5, 6)), (7, (8, 9))],
            dtype=[(str('A'), np.int64),
                   (str('B'), [(str('C'), np.int64), (str('D'), np.int64)])])
    
        tree = {'table_data': table}
    
        def check_raw_yaml(content):
            tree = yaml.load(
                re.sub(br'!core/\S+', b'', content))
    
            assert tree['table_data'] == {
                'datatype': [
                    {'datatype': 'int64', 'name': 'A', 'byteorder': 'little'},
                    {'datatype': [
                        {'datatype': 'int64', 'name': 'C', 'byteorder': 'little'},
                        {'datatype': 'int64', 'name': 'D', 'byteorder': 'little'}
                    ], 'name': 'B', 'byteorder': 'big'}],
                'shape': [3],
                'source': 0,
                'byteorder': 'big'
            }
    
>       helpers.assert_roundtrip_tree(tree, tmpdir, None, check_raw_yaml)

/usr/lib/python3/dist-packages/asdf/tags/core/tests/test_ndarray.py:250: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib/python3/dist-packages/asdf/tests/helpers.py:131: in assert_roundtrip_tree
    raw_yaml_check_func(content)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

content = b"%YAML 1.1\n%TAG ! tag:stsci.edu:asdf/\n--- !core/asdf-1.0.0\nasdf_library: !core/software-1.0.0 {author: Space Teles... int64, name: C}\n    - {byteorder: big, datatype: int64, name: D}\n    name: B\n  byteorder: big\n  shape: [3]\n...\n"

    def check_raw_yaml(content):
        tree = yaml.load(
            re.sub(br'!core/\S+', b'', content))
    
>       assert tree['table_data'] == {
            'datatype': [
                {'datatype': 'int64', 'name': 'A', 'byteorder': 'little'},
                {'datatype': [
                    {'datatype': 'int64', 'name': 'C', 'byteorder': 'little'},
                    {'datatype': 'int64', 'name': 'D', 'byteorder': 'little'}
                ], 'name': 'B', 'byteorder': 'big'}],
            'shape': [3],
            'source': 0,
            'byteorder': 'big'
        }
E       assert {'byteorder':..., 'source': 0} == {'byteorder': ..., 'source': 0}
E         Omitting 3 identical items, use -v to show
E         Differing items:
E         {'datatype': [{'byteorder': 'big', 'datatype': 'int64', 'name': 'A'}, {'byteorder': 'big', 'datatype': [{'byteorder': 'big', 'datatype': 'int64', 'name': 'C'}, {'byteorder': 'big', 'datatype': 'int64', 'name': 'D'}], 'name': 'B'}]} != {'datatype': [{'byteorder': 'little', 'datatype': 'int64', 'name': 'A'}, {'byteorder': 'big', 'datatype': [{'byteorder': 'little', 'datatype': 'int64', 'name': 'C'}, {'byteorder': 'little', 'datatype': 'int64', 'name': 'D'}], 'name': 'B'}]}
E         Use -v to get the full diff

/usr/lib/python3/dist-packages/asdf/tags/core/tests/test_ndarray.py:238: AssertionError
________________________________ test_checksum _________________________________

tmpdir = '/tmp/pytest-of-ubuntu/pytest-0/test_checksum0'

    def test_checksum(tmpdir):
        tmpdir = str(tmpdir)
        path = os.path.join(tmpdir, 'test.asdf')
    
        my_array = np.arange(0, 64, dtype=np.int64).reshape((8, 8))
        tree = {'my_array': my_array}
        ff = asdf.AsdfFile(tree)
        ff.write_to(path)
    
        with asdf.AsdfFile.open(path, validate_checksums=True) as ff:
            assert type(ff.blocks._internal_blocks[0].checksum) == bytes
>           assert ff.blocks._internal_blocks[0].checksum == \
                b'\xcaM\\\xb8t_L|\x00\n+\x01\xf1\xcfP1'
E           assert b'\xab\xc17\x...x97\x1b@\x86G' == b'\xcaM\\\xb8t...x01\xf1\xcfP1'
E             At index 0 diff: 171 != 202
E             Use -v to get the full diff

/usr/lib/python3/dist-packages/asdf/tests/test_low_level.py:740: AssertionError
_____________________________ test_checksum_update _____________________________

tmpdir = '/tmp/pytest-of-ubuntu/pytest-0/test_checksum_update0'

    def test_checksum_update(tmpdir):
        tmpdir = str(tmpdir)
        path = os.path.join(tmpdir, 'test.asdf')
    
        my_array = np.arange(0, 64, dtype=np.int64).reshape((8, 8))
    
        tree = {'my_array': my_array}
        ff = asdf.AsdfFile(tree)
        ff.write_to(path)
    
        with asdf.AsdfFile.open(path, mode='rw') as ff:
            ff.tree['my_array'][7, 7] = 0.0
            # update() should update the checksum, even if the data itself
            # is memmapped and isn't expressly re-written.
            ff.update()
    
        with asdf.AsdfFile.open(path, validate_checksums=True) as ff:
>           assert ff.blocks._internal_blocks[0].checksum == \
                b'T\xaf~[\x90\x8a\x88^\xc2B\x96D,N\xadL'
E           assert b'\x01\xf5\xb...x99`\xd7\xa9w' == b'T\xaf~[\x90\...B\x96D,N\xadL'
E             At index 0 diff: 1 != 84
E             Use -v to get the full diff

/usr/lib/python3/dist-packages/asdf/tests/test_low_level.py:761: AssertionError

[xnox]

command2-stdout.txt
command1-stdout.txt

Full test logs

[perrygreenfield]

We've been short handed for the past few months. Sorry for the late response. This is changing now, and work on supporting ASDF has picked up. I'm not sure how quickly we can get to this issue though since we have a backlog of issues to deal with.