diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4505f96..b52c3d7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: uses: mymindstorm/setup-emsdk@v12 with: # Make sure to set a version number! - version: 3.1.28 + version: 3.1.43 # This is the name of the cache folder. # The cache folder will be placed in the build directory, # so make sure it doesn't conflict with anything! @@ -36,5 +36,10 @@ jobs: npm i npm run build + - name: check environment + run: | + pwd + ls -al ./test + - name: test run: npm test diff --git a/src/hdf5_hl.d.ts b/src/hdf5_hl.d.ts index 2fab5eb..2bc247f 100644 --- a/src/hdf5_hl.d.ts +++ b/src/hdf5_hl.d.ts @@ -12,7 +12,7 @@ export declare const ACCESS_MODES: { readonly Sr: "H5F_ACC_SWMR_READ"; }; declare type ACCESS_MODESTRING = keyof typeof ACCESS_MODES; -export declare type OutputData = TypedArray | string | number | bigint | boolean | OutputData[]; +export declare type OutputData = TypedArray | string | number | bigint | boolean | Reference | RegionReference | OutputData[]; export declare type JSONCompatibleOutputData = string | number | boolean | JSONCompatibleOutputData[]; export declare type Dtype = string | { compound_type: CompoundTypeMetadata; @@ -29,13 +29,15 @@ declare type TypedArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Arra * `[i0, i1, s]` - select every `s` values in the range `i0` to `i1` **/ declare type Slice = [] | [number | null] | [number | null, number | null] | [number | null, number | null, number | null]; -export declare type GuessableDataTypes = TypedArray | number | number[] | string | string[]; +export declare type GuessableDataTypes = TypedArray | number | number[] | string | string[] | Reference | Reference[] | RegionReference | RegionReference[]; declare enum OBJECT_TYPE { DATASET = "Dataset", GROUP = "Group", BROKEN_SOFT_LINK = "BrokenSoftLink", EXTERNAL_LINK = "ExternalLink", - DATATYPE = "Datatype" + DATATYPE = "Datatype", + REFERENCE = "Reference", + REGION_REFERENCE = "RegionReference" } export declare class BrokenSoftLink { target: string; @@ -52,6 +54,12 @@ export declare class Datatype { type: OBJECT_TYPE; constructor(); } +export declare class Reference { + ref_data: Uint8Array; + constructor(ref_data: Uint8Array); +} +export declare class RegionReference extends Reference { +} export declare class Attribute { file_id: bigint; path: string; @@ -77,6 +85,7 @@ declare abstract class HasAttrs { get_attribute(name: string, json_compatible: false): OutputData; create_attribute(name: string, data: GuessableDataTypes, shape?: number[] | null, dtype?: string | null): void; delete_attribute(name: string): number; + create_reference(): Reference; } export declare class Group extends HasAttrs { constructor(file_id: bigint, path: string); @@ -90,6 +99,7 @@ export declare class Group extends HasAttrs { obj_path: string; }; get(obj_name: string): BrokenSoftLink | ExternalLink | Datatype | Group | Dataset | null; + dereference(ref: Reference | RegionReference): BrokenSoftLink | ExternalLink | Datatype | Group | Dataset | DatasetRegion | null; create_group(name: string): Group; create_dataset(args: { name: string; @@ -126,6 +136,7 @@ export declare class Dataset extends HasAttrs { get json_value(): JSONCompatibleOutputData; slice(ranges: Slice[]): OutputData; write_slice(ranges: Slice[], data: any): void; + create_region_reference(ranges: Slice[]): RegionReference; to_array(): string | number | boolean | JSONCompatibleOutputData[]; resize(new_shape: number[]): number; make_scale(scale_name?: string): void; @@ -137,11 +148,21 @@ export declare class Dataset extends HasAttrs { get_dimension_labels(): (string | null)[]; _value_getter(json_compatible?: boolean): OutputData; } +export declare class DatasetRegion { + source_dataset: Dataset; + region_reference: RegionReference; + private _metadata?; + constructor(source_dataset: Dataset, region_reference: RegionReference); + get metadata(): Metadata; + get value(): OutputData; + _value_getter(json_compatible?: boolean): OutputData; +} export declare const h5wasm: { File: typeof File; Group: typeof Group; Dataset: typeof Dataset; Datatype: typeof Datatype; + DatasetRegion: typeof DatasetRegion; ready: Promise; ACCESS_MODES: { readonly r: "H5F_ACC_RDONLY"; diff --git a/src/hdf5_hl.ts b/src/hdf5_hl.ts index 3ca82af..853245f 100644 --- a/src/hdf5_hl.ts +++ b/src/hdf5_hl.ts @@ -80,7 +80,7 @@ function getAccessor(type: 0 | 1, size: Metadata["size"], signed: Metadata["sign } } -export type OutputData = TypedArray | string | number | bigint | boolean | OutputData[]; +export type OutputData = TypedArray | string | number | bigint | boolean | Reference | RegionReference | OutputData[]; export type JSONCompatibleOutputData = string | number | boolean | JSONCompatibleOutputData[]; export type Dtype = string | {compound_type: CompoundTypeMetadata} | {array_type: Metadata}; export type { Metadata, Filter, CompoundMember, CompoundTypeMetadata, EnumTypeMetadata }; @@ -181,6 +181,16 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo } } + else if (type === Module.H5T_class_t.H5T_REFERENCE.value) { + const { ref_type, size } = metadata; // as { ref_type: 'object' | 'region', size: number }; + const cls = (ref_type === 'object') ? Reference : RegionReference; + output_data = Array.from({ length: metadata.total_size }).map((_, i) => { + const ref_data = data.slice(i*size, (i+1)*size); + return new cls(ref_data); + }); + return output_data; + } + else { known_type = false; output_data = data; @@ -275,6 +285,10 @@ function prepare_data(data: any, metadata: Metadata, shape?: number[] | bigint[] } output = new Uint8Array(typed_array.buffer); } + else if (metadata.type === Module.H5T_class_t.H5T_REFERENCE.value) { + output = new Uint8Array(metadata.size * total_size); + (data as Reference[]).forEach((r, i) => (output as Uint8Array).set(r.ref_data, i*metadata.size)); + } else { throw new Error(`data with type ${metadata.type} can not be prepared for write`); } @@ -316,35 +330,45 @@ function metadata_to_dtype(metadata: Metadata): Dtype { else if (type === Module.H5T_class_t.H5T_ARRAY.value ) { return { array_type: array_type as Metadata } } + else if (type === Module.H5T_class_t.H5T_REFERENCE.value) { + return (metadata.ref_type === 'object') ? "Reference" : "RegionReference"; + } else { return "unknown"; } } function dtype_to_metadata(dtype_str: string) { - let match = dtype_str.match(/^([<>|]?)([bhiqefdsBHIQS])([0-9]*)$/); - if (match == null) { - throw dtype_str + " is not a recognized dtype" - } - let [full, endianness, typestr, length] = match; let metadata = { vlen: false, signed: false } as Metadata; - metadata.littleEndian = (endianness != '>'); - if (fmts_int.has(typestr.toLowerCase())) { - metadata.type = Module.H5T_class_t.H5T_INTEGER.value; - metadata.size = (fmts_int.get(typestr.toLowerCase()) as number); - metadata.signed = (typestr.toLowerCase() == typestr); - } - else if (fmts_float.has(typestr)) { - metadata.type = Module.H5T_class_t.H5T_FLOAT.value; - metadata.size = (fmts_float.get(typestr) as number); - } - else if (typestr.toUpperCase() == 'S') { - metadata.type = Module.H5T_class_t.H5T_STRING.value; - metadata.size = (length == "") ? 4 : parseInt(length, 10); - metadata.vlen = (length == ""); + if (dtype_str === "Reference" || dtype_str === "RegionReference") { + metadata.type = Module.H5T_class_t.H5T_REFERENCE.value; + metadata.size = (dtype_str === "Reference") ? Module.SIZEOF_OBJ_REF : Module.SIZEOF_DSET_REGION_REF; + metadata.littleEndian = true; } else { - throw "should never happen" + let match = dtype_str.match(/^([<>|]?)([bhiqefdsBHIQS])([0-9]*)$/); + if (match == null) { + throw dtype_str + " is not a recognized dtype" + } + let [full, endianness, typestr, length] = match; + metadata.littleEndian = (endianness != '>'); + if (fmts_int.has(typestr.toLowerCase())) { + metadata.type = Module.H5T_class_t.H5T_INTEGER.value; + metadata.size = (fmts_int.get(typestr.toLowerCase()) as number); + metadata.signed = (typestr.toLowerCase() == typestr); + } + else if (fmts_float.has(typestr)) { + metadata.type = Module.H5T_class_t.H5T_FLOAT.value; + metadata.size = (fmts_float.get(typestr) as number); + } + else if (typestr.toUpperCase() === 'S') { + metadata.type = Module.H5T_class_t.H5T_STRING.value; + metadata.size = (length == "") ? 4 : parseInt(length, 10); + metadata.vlen = (length == ""); + } + else { + throw "should never happen" + } } return metadata } @@ -397,7 +421,7 @@ const TypedArray_to_dtype = new Map([ **/ type Slice = [] | [number|null] | [number|null,number|null] | [number|null, number|null, number|null]; -export type GuessableDataTypes = TypedArray | number | number[] | string | string[]; +export type GuessableDataTypes = TypedArray | number | number[] | string | string[] | Reference | Reference[] | RegionReference | RegionReference[]; function guess_dtype(data: GuessableDataTypes): string { if (ArrayBuffer.isView(data)) { @@ -417,7 +441,13 @@ function guess_dtype(data: GuessableDataTypes): string { return ' (typeof d == 'string'))) { - return 'S' + return 'S'; + } + else if (arr_data.every((d) => d instanceof RegionReference)) { + return 'RegionReference'; + } + else if (arr_data.every((d) => d instanceof Reference)) { + return 'Reference'; } } throw new Error("unguessable type for data"); @@ -428,7 +458,9 @@ enum OBJECT_TYPE { GROUP = "Group", BROKEN_SOFT_LINK = "BrokenSoftLink", EXTERNAL_LINK = "ExternalLink", - DATATYPE = 'Datatype' + DATATYPE = 'Datatype', + REFERENCE = 'Reference', + REGION_REFERENCE = 'RegionReference', } export class BrokenSoftLink { @@ -455,6 +487,16 @@ export class Datatype { constructor() {} } +export class Reference { + ref_data: Uint8Array; + constructor(ref_data: Uint8Array) { + this.ref_data = ref_data; + } +} + +export class RegionReference extends Reference { +} + export class Attribute { file_id: bigint; path: string; @@ -572,6 +614,10 @@ abstract class HasAttrs { return Module.delete_attribute(this.file_id, this.path, name); } + create_reference(): Reference { + const ref_data = Module.create_object_reference(this.file_id, this.path); + return new Reference(ref_data); + } } export class Group extends HasAttrs { @@ -640,6 +686,13 @@ export class Group extends HasAttrs { return null } + dereference(ref: Reference | RegionReference) { + const is_region = (ref instanceof RegionReference); + const name = Module.get_referenced_name(this.file_id, ref.ref_data, !is_region); + const target = this.get(name); + return (is_region) ? new DatasetRegion(target as Dataset, ref) : target; + } + create_group(name: string): Group { Module.create_group(this.file_id, this.path + "/" + name); return this.get(name) as Group; @@ -896,6 +949,15 @@ export class Dataset extends HasAttrs { } } + create_region_reference(ranges: Slice[]) { + const metadata = this.metadata; + // interpret ranges as [start, stop], with one per dim. + const { shape } = metadata; + const {strides, count, offset} = calculateHyperslabParams(shape, ranges); + const ref_data = Module.create_region_reference(this.file_id, this.path, count, offset, strides); + return new RegionReference(ref_data); + } + to_array() { const { json_value, metadata } = this; const { shape } = metadata; @@ -972,6 +1034,47 @@ export class Dataset extends HasAttrs { } +export class DatasetRegion { + source_dataset: Dataset; + region_reference: RegionReference; + private _metadata?: Metadata; + + constructor(source_dataset: Dataset, region_reference: RegionReference) { + this.source_dataset = source_dataset; + this.region_reference = region_reference; + } + + get metadata() { + if (typeof this._metadata === "undefined") { + this._metadata = Module.get_region_metadata(this.source_dataset.file_id, this.region_reference.ref_data); + } + return this._metadata; + } + + get value() { + return this._value_getter(false); + } + + _value_getter(json_compatible=false) { + let metadata = this.metadata; + // if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id; + let nbytes = metadata.size * metadata.total_size; + let data_ptr = Module._malloc(nbytes); + let processed: OutputData; + try { + Module.get_region_data(this.source_dataset.file_id, this.region_reference.ref_data, BigInt(data_ptr)); + let data = Module.HEAPU8.slice(data_ptr, data_ptr + nbytes); + processed = process_data(data, metadata, json_compatible); + } finally { + if (metadata.vlen) { + Module.reclaim_vlen_memory(this.source_dataset.file_id, this.source_dataset.path, "", BigInt(data_ptr)); + } + Module._free(data_ptr); + } + return processed; + } +} + function create_nested_array(value: JSONCompatibleOutputData[], shape: number[]) { // check that shapes match: const total_length = value.length; @@ -1001,6 +1104,7 @@ export const h5wasm = { Group, Dataset, Datatype, + DatasetRegion, ready, ACCESS_MODES } diff --git a/src/hdf5_util.cc b/src/hdf5_util.cc index 1f72ab9..37753d8 100644 --- a/src/hdf5_util.cc +++ b/src/hdf5_util.cc @@ -325,6 +325,11 @@ val get_dtype_metadata(hid_t dtype) enum_type.set("members", members); attr.set("enum_type", enum_type); } + else if (dtype_class == H5T_REFERENCE) + { + std::string ref_type = (H5Tequal(dtype, H5T_STD_REF_OBJ)) ? "object" : "region"; + attr.set("ref_type", ref_type); + } bool littleEndian = (order == H5T_ORDER_LE); attr.set("vlen", (bool)H5Tis_variable_str(dtype)); @@ -340,15 +345,15 @@ val get_abstractDS_metadata(hid_t dspace, hid_t dtype, hid_t dcpl) int rank = H5Sget_simple_extent_ndims(dspace); int total_size = H5Sget_simple_extent_npoints(dspace); - hsize_t dims_out[rank]; - hsize_t maxdims_out[rank]; - int ndims = H5Sget_simple_extent_dims(dspace, dims_out, maxdims_out); + std::vector dims_out(rank); + std::vector maxdims_out(rank); + int ndims = H5Sget_simple_extent_dims(dspace, dims_out.data(), maxdims_out.data()); val shape = val::array(); val maxshape = val::array(); for (int d = 0; d < ndims; d++) { - shape.set(d, (uint)dims_out[d]); - maxshape.set(d, (uint)maxdims_out[d]); + shape.set(d, (uint)dims_out.at(d)); + maxshape.set(d, (uint)maxdims_out.at(d)); } attr.set("shape", shape); @@ -358,12 +363,12 @@ val get_abstractDS_metadata(hid_t dspace, hid_t dtype, hid_t dcpl) if (dcpl) { H5D_layout_t layout = H5Pget_layout(dcpl); if (layout == H5D_CHUNKED) { - hsize_t chunk_dims_out[ndims]; - H5Pget_chunk(dcpl, ndims, chunk_dims_out); + std::vector chunk_dims_out(ndims); + H5Pget_chunk(dcpl, ndims, chunk_dims_out.data()); val chunks = val::array(); for (int c = 0; c < ndims; c++) { - chunks.set(c, (uint)chunk_dims_out[c]); + chunks.set(c, (uint)chunk_dims_out.at(c)); } attr.set("chunks", chunks); } @@ -523,9 +528,9 @@ int read_write_dataset_data(hid_t loc_id, const std::string& dataset_name_string if (count_out != val::null() && offset_out != val::null()) { - std::vector count = vecFromJSArray(count_out); - std::vector offset = vecFromJSArray(offset_out); - std::vector strides = vecFromJSArray(stride_out); + std::vector count = convertJSArrayToNumberVector(count_out); + std::vector offset = convertJSArrayToNumberVector(offset_out); + std::vector strides = convertJSArrayToNumberVector(stride_out); memspace = H5Screate_simple(count.size(), &count[0], nullptr); status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET, &offset[0], &strides[0], &count[0], NULL); @@ -701,6 +706,15 @@ herr_t setup_dataset(val dims_in, val maxdims_in, val chunks_in, int dtype, int throw_error("data type not supported"); } } + else if (dtype == H5T_REFERENCE) + { + if (dsize == sizeof(hobj_ref_t)) { + *filetype = H5Tcopy(H5T_STD_REF_OBJ); + } + else if (dsize == sizeof(hdset_reg_ref_t)) { + *filetype = H5Tcopy(H5T_STD_REF_DSETREG); + } + } else { throw_error("data type not supported"); @@ -1036,6 +1050,157 @@ val get_dimension_labels(hid_t loc_id, const std::string& target_dset_name_strin return dim_labels; } +// References +val create_object_reference(hid_t loc_id, const std::string& obj_name_string) +{ + const char *obj_name = obj_name_string.c_str(); + std::vector ref(sizeof(hobj_ref_t)); + herr_t status = H5Rcreate(ref.data(), loc_id, obj_name, H5R_OBJECT, (hid_t)-1); + return val::array(ref); +} + +val create_region_reference(hid_t loc_id, const std::string& dataset_name_string, val count_out, val offset_out, val stride_out) +{ + hid_t ds_id; + hid_t dspace; + std::vector ref(sizeof(hdset_reg_ref_t)); + herr_t status; + const char *dataset_name = dataset_name_string.c_str(); + + ds_id = H5Dopen2(loc_id, dataset_name, H5P_DEFAULT); + dspace = H5Dget_space(ds_id); + if (count_out != val::null() && offset_out != val::null()) + { + std::vector count = convertJSArrayToNumberVector(count_out); + std::vector offset = convertJSArrayToNumberVector(offset_out); + std::vector strides = convertJSArrayToNumberVector(stride_out); + status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET, &offset[0], &strides[0], &count[0], NULL); + } + else + { + status = H5Sselect_all(dspace); + } + status = H5Rcreate(ref.data(), loc_id, dataset_name, H5R_DATASET_REGION, dspace); + H5Sclose(dspace); + H5Dclose(ds_id); + return val::array(ref); +} + +val get_referenced_name(hid_t loc_id, const val ref_data_in, const bool is_object) +{ + ssize_t namesize = 0; + std::vector ref_data_vec = convertJSArrayToNumberVector(ref_data_in); + const hobj_ref_t *ref_ptr = (hobj_ref_t *)ref_data_vec.data(); + val output = val::null(); + const H5R_type_t ref_type = (is_object) ? H5R_OBJECT : H5R_DATASET_REGION; + hid_t object_id = H5Rdereference2(loc_id, H5P_DEFAULT, ref_type, ref_ptr); + namesize = H5Iget_name(object_id, nullptr, 0); + if (namesize > 0) + { + char *name = new char[namesize + 1]; + H5Iget_name(object_id, name, namesize + 1); + + output = val::u8string(name); + delete[] name; + } + H5Oclose(object_id); + return output; +} + +val get_region_metadata(hid_t loc_id, const val ref_data_in) +{ + hid_t dspace; + hid_t dtype; + hid_t dcpl; + herr_t status; + const std::vector ref_data_vec = convertJSArrayToNumberVector(ref_data_in); + const hdset_reg_ref_t *ref_ptr = (hdset_reg_ref_t *)ref_data_vec.data(); + hid_t ds_id = H5Rdereference2(loc_id, H5P_DEFAULT, H5R_DATASET_REGION, ref_ptr); + + dtype = H5Dget_type(ds_id); + dspace = H5Rget_region(ds_id, H5R_DATASET_REGION, ref_ptr); + dcpl = H5Dget_create_plist(ds_id); + // fill in shape, maxshape, chunks, total_size + val metadata = get_abstractDS_metadata(dspace, dtype, dcpl); + // then override the ones that are specific to a region: + int total_size = H5Sget_select_npoints(dspace); + metadata.set("total_size", total_size); + + int rank = H5Sget_simple_extent_ndims(dspace); + // shape will be null if the selection is not a regular hyperslab + val shape = val::null(); + htri_t is_regular = H5Sis_regular_hyperslab(dspace); + if (is_regular > 0) + { + std::vector count(rank); + std::vector block(rank); + htri_t success = H5Sget_regular_hyperslab(dspace, nullptr, nullptr, count.data(), block.data()); + shape = val::array(); + for (int d = 0; d < rank; d++) + { + int blocksize = (block.at(d) == NULL) ? 1 : block.at(d); + shape.set(d, (uint)(count.at(d) * blocksize)); + } + } + metadata.set("shape", shape); + H5Dclose(ds_id); + H5Sclose(dspace); + H5Tclose(dtype); + H5Pclose(dcpl); + return metadata; +} + +int get_region_data(hid_t loc_id, val ref_data_in, uint64_t rdata_uint64) +{ + hid_t ds_id; + hid_t dspace; + hid_t dtype; + hid_t memtype; + hid_t memspace; + herr_t status; + void *rdata = (void *)rdata_uint64; + const std::vector ref_data_vec = convertJSArrayToNumberVector(ref_data_in); + const hdset_reg_ref_t *ref_ptr = (hdset_reg_ref_t *)ref_data_vec.data(); + ds_id = H5Rdereference2(loc_id, H5P_DEFAULT, H5R_DATASET_REGION, ref_ptr); + dspace = H5Rget_region(ds_id, H5R_DATASET_REGION, ref_ptr); + dtype = H5Dget_type(ds_id); + // assumes that data to write will match type of dataset (exept endianness) + memtype = H5Tcopy(dtype); + // inputs and outputs from javascript will always be little-endian + H5T_order_t dorder = H5Tget_order(dtype); + if (dorder == H5T_ORDER_BE || dorder == H5T_ORDER_VAX) + { + status = H5Tset_order(memtype, H5T_ORDER_LE); + } + int rank = H5Sget_simple_extent_ndims(dspace); + htri_t is_regular = H5Sis_regular_hyperslab(dspace); + if (is_regular > 0) + { + std::vector count(rank); + std::vector block(rank); + std::vector shape_out(rank); + htri_t success = H5Sget_regular_hyperslab(dspace, nullptr, nullptr, count.data(), block.data()); + for (int d = 0; d < rank; d++) + { + int blocksize = (block.at(d) == NULL) ? 1 : block.at(d); + shape_out.at(d) = (count.at(d) * blocksize); + } + memspace = H5Screate_simple(shape_out.size(), &shape_out[0], nullptr); + } + else + { + hsize_t total_size = H5Sget_select_npoints(dspace); + memspace = H5Screate_simple(1, &total_size, nullptr); + } + status = H5Dread(ds_id, memtype, memspace, dspace, H5P_DEFAULT, rdata); + H5Dclose(ds_id); + H5Sclose(dspace); + H5Sclose(memspace); + H5Tclose(dtype); + H5Tclose(memtype); + return (int)status; +} + EMSCRIPTEN_BINDINGS(hdf5) { function("get_keys", &get_keys_vector); @@ -1074,6 +1239,11 @@ EMSCRIPTEN_BINDINGS(hdf5) function("get_attached_scales", &get_attached_scales); function("get_dimension_labels", &get_dimension_labels); function("set_dimension_label", &set_dimension_label); + function("create_object_reference", &create_object_reference); + function("create_region_reference", &create_region_reference); + function("get_referenced_name", &get_referenced_name); + function("get_region_metadata", &get_region_metadata); + function("get_region_data", &get_region_data); class_("H5L_info2_t") .constructor<>() @@ -1124,6 +1294,8 @@ EMSCRIPTEN_BINDINGS(hdf5) constant("H5O_TYPE_GROUP", (int)H5O_TYPE_GROUP); constant("H5O_TYPE_DATASET", (int)H5O_TYPE_DATASET); constant("H5O_TYPE_NAMED_DATATYPE", (int)H5O_TYPE_NAMED_DATATYPE); + constant("SIZEOF_OBJ_REF", (int)(sizeof(hobj_ref_t))); + constant("SIZEOF_DSET_REGION_REF", (int)(sizeof(hdset_reg_ref_t))); constant("H5Z_FILTER_NONE", H5Z_FILTER_NONE); constant("H5Z_FILTER_DEFLATE", H5Z_FILTER_DEFLATE); diff --git a/src/hdf5_util_helpers.d.ts b/src/hdf5_util_helpers.d.ts index d252517..8bba65e 100644 --- a/src/hdf5_util_helpers.d.ts +++ b/src/hdf5_util_helpers.d.ts @@ -25,6 +25,7 @@ export interface Metadata { enum_type?: EnumTypeMetadata, littleEndian: boolean, maxshape: Array | null, + ref_type?: 'object' | 'region', shape: Array, signed: boolean, size: number, @@ -59,6 +60,8 @@ export interface H5Module extends EmscriptenModule { get_external_link(file_id: bigint, obj_path: string): {filename: string, obj_path: string}; H5O_TYPE_DATASET: number; H5O_TYPE_GROUP: number; + SIZEOF_OBJ_REF: number; + SIZEOF_DSET_REGION_REF: number; H5G_GROUP: number; H5G_DATASET: number; H5G_TYPE: number; @@ -114,6 +117,11 @@ export interface H5Module extends EmscriptenModule { get_attached_scales(loc_id: bigint, target_dset_name: string, index: number): string[], set_dimension_label(loc_id: bigint, target_dset_name: string, index: number, label: string): number, get_dimension_labels(loc_id: bigint, target_dset_name: string): Array, + create_object_reference(loc_id: bigint, target_name: string): Uint8Array, + create_region_reference(file_id: bigint, path: string, count: bigint[] | null, offset: bigint[] | null, strides: bigint[] | null): Uint8Array, + get_referenced_name(loc_id: bigint, ref_ptr: Uint8Array, is_object: boolean): string; + get_region_metadata(loc_id: bigint, ref_ptr: Uint8Array): Metadata; + get_region_data(loc_id: bigint, ref_data: Uint8Array, rdata_ptr: bigint): number; } export declare type Filter = { diff --git a/test/create_read_references.mjs b/test/create_read_references.mjs new file mode 100644 index 0000000..f4da521 --- /dev/null +++ b/test/create_read_references.mjs @@ -0,0 +1,96 @@ +#!/usr/bin/env node + +import { strict as assert } from 'assert'; +import { existsSync, mkdirSync, unlinkSync } from 'fs'; +import { join } from 'path'; +import h5wasm from "h5wasm"; + +async function test_refs() { + + await h5wasm.ready; + const PATH = join(".", "test", "tmp"); + const FILEPATH = join(PATH, "refs.h5"); + const VALUES = [12,11,10,9,8,7,6,5,4,3,2,1]; + const DATA = new Float32Array(VALUES); + const SHAPE = [4,3]; + const DATASET_GROUP = "entry"; + const DATASET_NAME = "data"; + const REFS_GROUP = "refs"; + const OBJECT_REF_DATASET_NAME = "object_refs"; + const REGION_REF_DATASET_NAME = "dset_region_refs"; + const REGION_REF_DATA_0 = [[11.], [ 8.], [ 5.]]; + const REGION_REF_DATA_1 = [[12., 11., 10.]]; + + if (!(existsSync(PATH))) { + mkdirSync(PATH); + } + + // write + { + const write_file = new h5wasm.File(FILEPATH, "w"); + + write_file.create_group(DATASET_GROUP); + const dataset_group = write_file.get(DATASET_GROUP); + dataset_group.create_dataset({name: DATASET_NAME, data: DATA, shape: SHAPE}); + + const object_refs = [ + dataset_group.create_reference(), + dataset_group.get(DATASET_NAME).create_reference(), + ]; + + write_file.create_group(REFS_GROUP); + const refs_group = write_file.get(REFS_GROUP); + refs_group.create_dataset({name: OBJECT_REF_DATASET_NAME, data: object_refs}); + + const dataset = dataset_group.get(DATASET_NAME); + const region_refs = [ + dataset.create_region_reference([[0,3], [1,2]]), + dataset.create_region_reference([[0,1], []]), + ] + refs_group.create_dataset({name: REGION_REF_DATASET_NAME, data: region_refs}) + + write_file.flush(); + write_file.close(); + } + + // read + { + const read_file = new h5wasm.File(FILEPATH, "r"); + + const dataset_group = read_file.get(DATASET_GROUP); + assert(dataset_group instanceof h5wasm.Group); + + const refs_group = read_file.get(REFS_GROUP); + assert(refs_group instanceof h5wasm.Group); + + const object_refs = refs_group.get(OBJECT_REF_DATASET_NAME).value; + const [obj_0, obj_1] = object_refs.map((ref) => read_file.dereference(ref)); + assert(obj_0 instanceof h5wasm.Group); + assert.strictEqual(obj_0.path, `/${DATASET_GROUP}`); + assert(obj_1 instanceof h5wasm.Dataset); + assert.strictEqual(obj_1.path, `/${DATASET_GROUP}/${DATASET_NAME}`); + + const region_refs = refs_group.get(REGION_REF_DATASET_NAME).value; + const [region_0, region_1] = region_refs.map((ref) => read_file.dereference(ref)); + assert(region_0 instanceof h5wasm.DatasetRegion); + assert.deepEqual(region_0.value, new Float32Array(REGION_REF_DATA_0.flat())); + assert(region_1 instanceof h5wasm.DatasetRegion); + assert.deepEqual(region_1.value, new Float32Array(REGION_REF_DATA_1.flat())); + // assert.deepEqual(hard_link_dataset.value, DATA); + + read_file.close() + } + + // cleanup file when finished: + unlinkSync(FILEPATH); + +} + +export const tests = [ + { + description: "Create and read object and region references", + test: test_refs + }, +]; + +export default tests; diff --git a/test/test.mjs b/test/test.mjs index b7decea..ae3563b 100644 --- a/test/test.mjs +++ b/test/test.mjs @@ -15,6 +15,7 @@ import bigendian_read from './bigendian_read.mjs'; import create_compressed from './create_read_compressed.mjs'; import dimension_labels from './dimension_labels.mjs'; import dimension_scales from './dimension_scales.mjs'; +import references from './create_read_references.mjs'; let tests = []; const add_tests = (tests_in) => { /*global*/ tests = tests.concat(tests_in)} @@ -33,7 +34,9 @@ add_tests(bigendian_read); add_tests(create_compressed); add_tests(dimension_labels); add_tests(dimension_scales); +add_tests(references); +let passed = true; async function run_test(test) { try { await test.test(); @@ -42,6 +45,7 @@ async function run_test(test) { catch (error) { console.log('x', test.description); console.log(error.stack); + passed = false; } } @@ -52,3 +56,6 @@ async function run_tests(tests) { } await run_tests(tests); +if (!passed) { + throw new Error("Tests did not complete successfuly"); +}