diff --git a/src/hdf5_hl.d.ts b/src/hdf5_hl.d.ts index 07b8ae6..109b793 100644 --- a/src/hdf5_hl.d.ts +++ b/src/hdf5_hl.d.ts @@ -66,13 +66,13 @@ export declare class Attribute { name: string; metadata: Metadata; dtype: Dtype; - shape: number[]; + shape: number[] | null; private _value?; private _json_value?; constructor(file_id: bigint, path: string, name: string); - get value(): OutputData; - get json_value(): JSONCompatibleOutputData; - to_array(): string | number | boolean | JSONCompatibleOutputData[]; + get value(): OutputData | null; + get json_value(): JSONCompatibleOutputData | null; + to_array(): JSONCompatibleOutputData | null; } declare abstract class HasAttrs { file_id: bigint; @@ -132,14 +132,14 @@ export declare class Dataset extends HasAttrs { refresh(): void; get metadata(): Metadata; get dtype(): Dtype; - get shape(): number[]; + get shape(): number[] | null; get filters(): Filter[]; - get value(): OutputData; - get json_value(): JSONCompatibleOutputData; - slice(ranges: Slice[]): OutputData; + get value(): OutputData | null; + get json_value(): JSONCompatibleOutputData | null; + slice(ranges: Slice[]): OutputData | null; write_slice(ranges: Slice[], data: any): void; create_region_reference(ranges: Slice[]): RegionReference; - to_array(): string | number | boolean | JSONCompatibleOutputData[]; + to_array(): JSONCompatibleOutputData | null; resize(new_shape: number[]): number; make_scale(scale_name?: string): void; attach_scale(index: number, scale_dset_path: string): void; @@ -148,7 +148,9 @@ export declare class Dataset extends HasAttrs { get_scale_name(): string | null; set_dimension_label(index: number, label: string): void; get_dimension_labels(): (string | null)[]; - _value_getter(json_compatible?: boolean): OutputData; + _value_getter(json_compatible?: false): OutputData | null; + _value_getter(json_compatible: true): JSONCompatibleOutputData | null; + _value_getter(json_compatible: boolean): OutputData | JSONCompatibleOutputData | null; } export declare class DatasetRegion { source_dataset: Dataset; @@ -156,8 +158,10 @@ export declare class DatasetRegion { private _metadata?; constructor(source_dataset: Dataset, region_reference: RegionReference); get metadata(): Metadata; - get value(): OutputData; - _value_getter(json_compatible?: boolean): OutputData; + get value(): OutputData | null; + _value_getter(json_compatible?: false): OutputData | null; + _value_getter(json_compatible: true): JSONCompatibleOutputData | null; + _value_getter(json_compatible: boolean): OutputData | JSONCompatibleOutputData | null; } export declare const h5wasm: { File: typeof File; diff --git a/src/hdf5_hl.ts b/src/hdf5_hl.ts index 1fcb494..e862e7c 100644 --- a/src/hdf5_hl.ts +++ b/src/hdf5_hl.ts @@ -40,11 +40,15 @@ function dirname(path: string) { return head; } -function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: true): JSONCompatibleOutputData; -function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: false): OutputData; -function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: boolean): OutputData | JSONCompatibleOutputData; -function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: boolean = false) { +function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: true): JSONCompatibleOutputData | null; +function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: false): OutputData | null; +function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: boolean): OutputData | JSONCompatibleOutputData | null; +function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: boolean = false): OutputData | JSONCompatibleOutputData | null { let metadata = Module.get_attribute_metadata(file_id, obj_name, attr_name); + if (!metadata.shape) { + return null; + } + let nbytes = metadata.size * metadata.total_size; let data_ptr = Module._malloc(nbytes); var processed; @@ -106,6 +110,7 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo // but otherwise returns Uint8Array raw bytes as loaded. let output_data: OutputData; let { shape, type } = metadata; + let known_type = true; // let length: number; if (type === Module.H5T_class_t.H5T_STRING.value) { @@ -153,9 +158,9 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo else if (type === Module.H5T_class_t.H5T_COMPOUND.value) { const { size, compound_type } = <{size: Metadata["size"], compound_type: CompoundTypeMetadata}>metadata; let n = Math.floor(data.byteLength / size); - let output: OutputData[] = []; + let output: (OutputData | JSONCompatibleOutputData)[] = []; for (let i = 0; i < n; i++) { - let row: OutputData = []; + let row: (OutputData | JSONCompatibleOutputData)[] = []; let row_data = data.slice(i * size, (i + 1) * size); for (let member of compound_type.members) { let member_data = row_data.slice(member.offset, member.offset + member.size); @@ -168,7 +173,7 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo else if (type === Module.H5T_class_t.H5T_ARRAY.value) { const { array_type } = <{array_type: Metadata}>metadata; - shape = shape.concat(array_type.shape); + shape = (shape).concat(array_type.shape); array_type.shape = shape; // always convert ARRAY types to base JS types: output_data = process_data(data, array_type, true); @@ -220,7 +225,7 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo } function isIterable(x: any): x is Iterable { - return typeof x === 'object' && Symbol.iterator in x; + return typeof x === 'object' && x !== null && Symbol.iterator in x; } function isH5PYBooleanEnum(enum_type: EnumTypeMetadata) { @@ -515,9 +520,9 @@ export class Attribute { name: string; metadata: Metadata; dtype: Dtype; - shape: number[]; - private _value?: OutputData; - private _json_value?: JSONCompatibleOutputData; + shape: number[] | null; + private _value?: OutputData | null; + private _json_value?: JSONCompatibleOutputData | null; constructor(file_id: bigint, path: string, name: string) { this.file_id = file_id; @@ -529,27 +534,27 @@ export class Attribute { this.shape = metadata.shape; } - get value() { + get value(): OutputData | null { if (typeof this._value === "undefined") { this._value = get_attr(this.file_id, this.path, this.name, false); } return this._value; } - get json_value() { + get json_value(): JSONCompatibleOutputData | null { if (typeof this._json_value === "undefined") { this._json_value = get_attr(this.file_id, this.path, this.name, true); } return this._json_value; } - to_array() { + to_array(): JSONCompatibleOutputData | null { const { json_value, metadata } = this; const { shape } = metadata; if (!isIterable(json_value) || typeof json_value === "string") { return json_value; } - return create_nested_array(json_value, shape); + return create_nested_array(json_value, shape); } } @@ -916,24 +921,28 @@ export class Dataset extends HasAttrs { return Module.get_dataset_filters(this.file_id, this.path); } - get value() { + get value(): OutputData | null { return this._value_getter(false); } - get json_value(): JSONCompatibleOutputData { - return this._value_getter(true) as JSONCompatibleOutputData; + get json_value(): JSONCompatibleOutputData | null { + return this._value_getter(true); } - slice(ranges: Slice[]) { + slice(ranges: Slice[]): OutputData | null { // interpret ranges as [start, stop], with one per dim. const metadata = this.metadata; // if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id; const { shape } = metadata; + if (!shape) { + return null; + } + const {strides, count, offset} = calculateHyperslabParams(shape, ranges); const total_size = count.reduce((previous, current) => current * previous, 1n); const nbytes = metadata.size * Number(total_size); const data_ptr = Module._malloc(nbytes); - let processed; + let processed: OutputData; try { Module.get_dataset_data(this.file_id, this.path, count, offset, strides, BigInt(data_ptr)); let data = Module.HEAPU8.slice(data_ptr, data_ptr + nbytes); @@ -950,6 +959,9 @@ export class Dataset extends HasAttrs { write_slice(ranges: Slice[], data: any) { // interpret ranges as [start, stop], with one per dim. let metadata = this.metadata; + if (!metadata.shape) { + throw new Error("cannot write to a slice of an empty dataset"); + } if (metadata.vlen) { throw new Error("writing to a slice of vlen dtype is not implemented"); } @@ -971,6 +983,10 @@ export class Dataset extends HasAttrs { create_region_reference(ranges: Slice[]) { const metadata = this.metadata; + if (!metadata.shape) { + throw new Error("cannot create region reference from empty dataset"); + } + // interpret ranges as [start, stop], with one per dim. const { shape } = metadata; const {strides, count, offset} = calculateHyperslabParams(shape, ranges); @@ -978,13 +994,13 @@ export class Dataset extends HasAttrs { return new RegionReference(ref_data); } - to_array() { + to_array(): JSONCompatibleOutputData | null { const { json_value, metadata } = this; const { shape } = metadata; if (!isIterable(json_value) || typeof json_value === "string") { return json_value; } - let nested = create_nested_array(json_value, shape); + let nested = create_nested_array(json_value, shape); return nested; } @@ -1033,8 +1049,15 @@ export class Dataset extends HasAttrs { return Module.get_dimension_labels(this.file_id, this.path); } - _value_getter(json_compatible=false) { + _value_getter(json_compatible?: false): OutputData | null; + _value_getter(json_compatible: true): JSONCompatibleOutputData | null; + _value_getter(json_compatible: boolean): OutputData | JSONCompatibleOutputData | null; + _value_getter(json_compatible=false): OutputData | JSONCompatibleOutputData | null { let metadata = this.metadata; + if (!metadata.shape) { + return null + } + // if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id; let nbytes = metadata.size * metadata.total_size; let data_ptr = Module._malloc(nbytes); @@ -1071,12 +1094,19 @@ export class DatasetRegion { return this._metadata; } - get value() { + get value(): OutputData | null { return this._value_getter(false); } - _value_getter(json_compatible=false) { + _value_getter(json_compatible?: false): OutputData | null; + _value_getter(json_compatible: true): JSONCompatibleOutputData | null; + _value_getter(json_compatible: boolean): OutputData | JSONCompatibleOutputData | null; + _value_getter(json_compatible=false): OutputData | JSONCompatibleOutputData | null { let metadata = this.metadata; + if (!metadata.shape) { + return null; + } + // if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id; let nbytes = metadata.size * metadata.total_size; let data_ptr = Module._malloc(nbytes); @@ -1108,7 +1138,7 @@ function create_nested_array(value: JSONCompatibleOutputData[], shape: number[]) const subdims = shape.slice(1).reverse(); for (let dim of subdims) { // in each pass, replace input with array of slices of input - const new_output: JSONCompatibleOutputData = []; + const new_output: JSONCompatibleOutputData[][] = []; const { length } = output; let cursor = 0; while (cursor < length) { diff --git a/src/hdf5_util.cc b/src/hdf5_util.cc index 3ea3ea9..1d62227 100644 --- a/src/hdf5_util.cc +++ b/src/hdf5_util.cc @@ -345,15 +345,26 @@ val get_abstractDS_metadata(hid_t dspace, hid_t dtype, hid_t dcpl) { val attr = get_dtype_metadata(dtype); - int rank = H5Sget_simple_extent_ndims(dspace); + int type = H5Sget_simple_extent_type(dspace); int total_size = H5Sget_simple_extent_npoints(dspace); + attr.set("total_size", total_size); + + if (type == H5S_NULL) { + attr.set("shape", val::null()); + attr.set("maxshape", val::null()); + attr.set("chunks", val::null()); + return attr; + } + + int rank = H5Sget_simple_extent_ndims(dspace); std::vector dims_out(rank); std::vector maxdims_out(rank); + int ndims = H5Sget_simple_extent_dims(dspace, dims_out.data(), maxdims_out.data()); + val shape = val::array(); val maxshape = val::array(); - for (int d = 0; d < ndims; d++) - { + for (int d = 0; d < ndims; d++) { shape.set(d, (uint)dims_out.at(d)); maxshape.set(d, (uint)maxdims_out.at(d)); } @@ -364,19 +375,20 @@ val get_abstractDS_metadata(hid_t dspace, hid_t dtype, hid_t dcpl) if (dcpl) { H5D_layout_t layout = H5Pget_layout(dcpl); + if (layout == H5D_CHUNKED) { std::vector chunk_dims_out(ndims); H5Pget_chunk(dcpl, ndims, chunk_dims_out.data()); + val chunks = val::array(); - for (int c = 0; c < ndims; c++) - { + for (int c = 0; c < ndims; c++) { chunks.set(c, (uint)chunk_dims_out.at(c)); } + attr.set("chunks", chunks); } } - attr.set("total_size", total_size); return attr; } diff --git a/src/hdf5_util_helpers.d.ts b/src/hdf5_util_helpers.d.ts index 08c85af..f41ab77 100644 --- a/src/hdf5_util_helpers.d.ts +++ b/src/hdf5_util_helpers.d.ts @@ -19,14 +19,14 @@ export interface H5T_class_t { export interface Metadata { array_type?: Metadata, - chunks: Array | null, + chunks: number[] | null, compound_type?: CompoundTypeMetadata, cset: number, enum_type?: EnumTypeMetadata, littleEndian: boolean, - maxshape: Array | null, + maxshape: number[] | null, ref_type?: 'object' | 'region', - shape: Array, + shape: number[] | null, signed: boolean, size: number, total_size: number, @@ -40,7 +40,7 @@ export interface CompoundMember extends Metadata { } export interface CompoundTypeMetadata { - members: Array + members: CompoundMember[] nmembers: number; } @@ -105,7 +105,7 @@ export interface H5Module extends EmscriptenModule { reclaim_vlen_memory(file_id: BigInt, obj_name: string, attr_name: string, data_ptr: bigint): Status; get_attribute_data(file_id: BigInt, obj_name: string, attr_name: string, arg3: bigint): Status; FS: FS.FileSystemType, - get_keys_vector(group_id: bigint, H5_index_t: number): Array, + get_keys_vector(group_id: bigint, H5_index_t: number): string[], get_attribute_metadata(loc_id: bigint, group_name_string: string, attribute_name_string: string): Metadata, get_plugin_search_paths(): string[], insert_plugin_search_path(search_path: string, index: number): number, @@ -116,7 +116,7 @@ export interface H5Module extends EmscriptenModule { get_scale_name(loc_id: bigint, dimscale_dset_name: string): string | null, get_attached_scales(loc_id: bigint, target_dset_name: string, index: number): string[], set_dimension_label(loc_id: bigint, target_dset_name: string, index: number, label: string): number, - get_dimension_labels(loc_id: bigint, target_dset_name: string): Array, + get_dimension_labels(loc_id: bigint, target_dset_name: string): (string | null)[], create_object_reference(loc_id: bigint, target_name: string): Uint8Array, create_region_reference(file_id: bigint, path: string, count: bigint[] | null, offset: bigint[] | null, strides: bigint[] | null): Uint8Array, get_referenced_name(loc_id: bigint, ref_ptr: Uint8Array, is_object: boolean): string;