Skip to content

Commit

Permalink
Add support for serializing big-endian arrays to Parquet.
Browse files Browse the repository at this point in the history
  • Loading branch information
erykoff committed Feb 8, 2023
1 parent 723bb16 commit fb89807
Showing 1 changed file with 14 additions and 1 deletion.
15 changes: 14 additions & 1 deletion astropy/io/misc/parquet.py
Expand Up @@ -450,7 +450,20 @@ def write_table_parquet(table, output, overwrite=False):
val = []
else:
val = encode_table[name]
arrays.append(pa.array(val, type=schema.field(name).type))
try:
arrays.append(pa.array(val, type=schema.field(name).type))
except pa.ArrowNotImplementedError as err:
# check if val is big-endian.
if (np.little_endian and val.dtype.byteorder == ">") or (
not np.little_endian and val.dtype.byteorder == "="
):
# We need to convert the array to little-endian.
val2 = val.byteswap()
val2.dtype = val2.dtype.newbyteorder("<")
arrays.append(pa.array(val2, type=schema.field(name).type))
else:
# This failed for some other reason, so raise.
raise err

# Create a pyarrow table from the list of arrays and the schema
pa_table = pa.Table.from_arrays(arrays, schema=schema)
Expand Down

0 comments on commit fb89807

Please sign in to comment.