Skip to content

[5.x] Zarr reads #802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lesserwhirls merged 5 commits into from
Aug 25, 2021
Merged

[5.x] Zarr reads #802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
73f29bb
base Zarr read support
Jun 28, 2021
a53f436
f order array support
Jul 26, 2021
664ebeb
finish tests; fix f order; implement nested store
Aug 17, 2021
0d0952a
spotless
Aug 24, 2021
8c43855
fix failing tests
Aug 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
81 changes: 74 additions & 7 deletions cdm/core/src/main/java/ucar/nc2/iosp/IospHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,27 +301,37 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
if (showLayoutTypes)
System.out.println("***BB LayoutType=" + layout.getClass().getName());

if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR)) {
if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR) || dataType == DataType.BOOLEAN) {
byte[] pa = (byte[]) arr;
while (layout.hasNext()) {
LayoutBB.Chunk chunk = layout.next();
ByteBuffer bb = chunk.getByteBuffer();
// if chunk is empty, use fill value
if (!bb.hasRemaining()) {
continue;
}
bb.position(chunk.getSrcElem());
int pos = (int) chunk.getDestElem();
for (int i = 0; i < chunk.getNelems(); i++)
pa[pos++] = bb.get();
}
// return (dataType == DataType.CHAR) ? convertByteToChar(pa) : pa;
if (dataType == DataType.CHAR)
if (dataType == DataType.CHAR) {
return convertByteToChar(pa);
else
} else if (dataType == DataType.BOOLEAN) {
return convertByteToBoolean(pa);
} else
return pa;

} else if (dataType.getPrimitiveClassType() == short.class) {
short[] pa = (short[]) arr;
while (layout.hasNext()) {
LayoutBB.Chunk chunk = layout.next();
ShortBuffer buff = chunk.getShortBuffer();
// if chunk is empty, use fill value
if (!buff.hasRemaining()) {
continue;
}
buff.position(chunk.getSrcElem());
int pos = (int) chunk.getDestElem();
for (int i = 0; i < chunk.getNelems(); i++)
Expand All @@ -334,6 +344,10 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
while (layout.hasNext()) {
LayoutBB.Chunk chunk = layout.next();
IntBuffer buff = chunk.getIntBuffer();
// if chunk is empty, use fill value
if (!buff.hasRemaining()) {
continue;
}
buff.position(chunk.getSrcElem());
int pos = (int) chunk.getDestElem();
for (int i = 0; i < chunk.getNelems(); i++)
Expand All @@ -346,6 +360,10 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
while (layout.hasNext()) {
LayoutBB.Chunk chunk = layout.next();
FloatBuffer buff = chunk.getFloatBuffer();
// if chunk is empty, use fill value
if (!buff.hasRemaining()) {
continue;
}
buff.position(chunk.getSrcElem());
int pos = (int) chunk.getDestElem();
for (int i = 0; i < chunk.getNelems(); i++)
Expand All @@ -358,6 +376,10 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
while (layout.hasNext()) {
LayoutBB.Chunk chunk = layout.next();
DoubleBuffer buff = chunk.getDoubleBuffer();
// if chunk is empty, use fill value
if (!buff.hasRemaining()) {
continue;
}
buff.position(chunk.getSrcElem());
int pos = (int) chunk.getDestElem();
for (int i = 0; i < chunk.getNelems(); i++)
Expand All @@ -370,6 +392,10 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
while (layout.hasNext()) {
LayoutBB.Chunk chunk = layout.next();
LongBuffer buff = chunk.getLongBuffer();
// if chunk is empty, use fill value
if (!buff.hasRemaining()) {
continue;
}
buff.position(chunk.getSrcElem());
int pos = (int) chunk.getDestElem();
for (int i = 0; i < chunk.getNelems(); i++)
Expand All @@ -383,12 +409,37 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
while (layout.hasNext()) {
LayoutBB.Chunk chunk = layout.next();
ByteBuffer bb = chunk.getByteBuffer();
// if chunk is empty, use fill value
if (!bb.hasRemaining()) {
continue;
}
bb.position(chunk.getSrcElem() * recsize);
int pos = (int) chunk.getDestElem() * recsize;
for (int i = 0; i < chunk.getNelems() * recsize; i++)
pa[pos++] = bb.get();
}
return pa;
} else if (dataType == DataType.STRING) {
String[] pa = (String[]) arr;
int recsize = layout.getElemSize();
while (layout.hasNext()) {
LayoutBB.Chunk chunk = layout.next();
ByteBuffer bb = chunk.getByteBuffer();
// if chunk is empty, use fill value
if (!bb.hasRemaining()) {
continue;
}
bb.position(chunk.getSrcElem() * recsize);
int pos = (int) chunk.getDestElem();
for (int i = 0; i < chunk.getNelems(); i++) {
char[] ch = new char[dataType.getSize()];
for (int j = 0; j < ch.length; j++) {
ch[j] = (char) bb.get();
}
pa[pos++] = new String(ch);
}
}
return pa;
}

throw new IllegalStateException();
Expand Down Expand Up @@ -666,9 +717,15 @@ public static Object makePrimitiveArray(int size, DataType dataType) {
*/
public static Object makePrimitiveArray(int size, DataType dataType, Object fillValue) {

if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR)) {
if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR) || dataType == DataType.BOOLEAN) {
byte[] pa = new byte[size];
byte val = ((Number) fillValue).byteValue();
byte val;
if (fillValue instanceof String) {
byte[] bytes = ((String) fillValue).getBytes();
val = bytes.length > 0 ? ((String) fillValue).getBytes()[0] : 0;
} else {
val = ((Number) fillValue).byteValue();
}
if (val != 0)
for (int i = 0; i < size; i++)
pa[i] = val;
Expand Down Expand Up @@ -755,7 +812,6 @@ public static byte[] convertCharToByteUTF(char[] from) {
}

// convert byte array to char array

public static char[] convertByteToChar(byte[] byteArray) {
int size = byteArray.length;
char[] cbuff = new char[size];
Expand All @@ -765,7 +821,6 @@ public static char[] convertByteToChar(byte[] byteArray) {
}

// convert char array to byte array

public static byte[] convertCharToByte(char[] from) {
byte[] to = null;
if (from != null) {
Expand All @@ -777,6 +832,18 @@ public static byte[] convertCharToByte(char[] from) {
return to;
}

// convert byte array to boolean array
public static boolean[] convertByteToBoolean(byte[] from) {
boolean[] to = null;
if (from != null) {
int size = from.length;
to = new boolean[size];
for (int i = 0; i < size; i++)
to[i] = from[i] != 0;
}
return to;
}

public static long transferData(Array result, WritableByteChannel channel) throws java.io.IOException {

// LOOK should we buffer ??
Expand Down
12 changes: 7 additions & 5 deletions cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZArray.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ public enum Order {
// .zarray fields
private final int[] shape;
private final int[] chunks;
private final Number fillValue;
private final Object fillValue;
private final DataType datatype;
private final String dtype;
private final ZarrFilter compressor;
Expand All @@ -69,7 +69,7 @@ public enum Order {
private final List<ZarrFilter> filters;
private final String separator;

public ZArray(int[] shape, int[] chunks, Number fill_value, String dtype, ZarrFilter compressor, String order,
public ZArray(int[] shape, int[] chunks, Object fill_value, String dtype, ZarrFilter compressor, String order,
List<ZarrFilter> filters, String separator) throws ZarrFormatException {
this.shape = shape;
this.chunks = chunks;
Expand Down Expand Up @@ -99,7 +99,7 @@ public List<ZarrFilter> getFilters() {
return this.filters;
}

public Number getFillValue() {
public Object getFillValue() {
return fillValue;
}

Expand Down Expand Up @@ -176,13 +176,15 @@ public ZArray deserialize(JsonParser p, DeserializationContext ctxt) throws IOEx
.mapToInt(JsonNode::asInt).toArray();
String dtype = ((JsonNode) root.path(ZarrKeys.DTYPE)).asText();
JsonNode fillValueNode = (JsonNode) root.path(ZarrKeys.FILL_VALUE);
final Number fill;
final Object fill;
if (fillValueNode.isLong()) {
fill = fillValueNode.longValue();
} else if (fillValueNode.isFloat()) {
fill = fillValueNode.floatValue();
} else {
} else if (fillValueNode.isNumber()) {
fill = fillValueNode.asDouble();
} else {
fill = fillValueNode.asText("");
}

String order = ((JsonNode) root.path(ZarrKeys.ORDER)).asText();
Expand Down
3 changes: 3 additions & 0 deletions cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrFormatException.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,7 @@ public ZarrFormatException(String invalidField, String value) {
super(String.format(customExceptionMessageFormat, invalidField, value));
}

public ZarrFormatException(String msg) {
super(msg);
}
}
Loading