hdf5: add support for chunk cache on a per-dataset basis

* Add api OpenDatasetWith

* Support chunk cache

* Rename variables' name

* Fix typos

* Remove unnecessary if condition

* Catch returned error

* Rename variables

* Update comments

* Fix bug to return error

h5d_public.go

@@ -0,0 +1,16 @@
+// Copyright ©2019 The Gonum Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package hdf5
+
+// #include "hdf5.h"
+import "C"
+
+// Used to unset chunk cache configuration parameter.
+// https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetChunkCache
+const (
+	D_CHUNK_CACHE_NSLOTS_DEFAULT int     = -1 // The number of chunk slots in the raw data chunk cache for this dataset
+	D_CHUNK_CACHE_NBYTES_DEFAULT int     = -1 // The total size of the raw data chunk cache for this dataset
+	D_CHUNK_CACHE_W0_DEFAULT     float64 = -1 // The chunk preemption policy for this dataset
+)

h5g_group.go

@@ -108,6 +108,19 @@ func (g *CommonFG) OpenDataset(name string) (*Dataset, error) {
 	return newDataset(hid, nil), nil
 }
 
+// OpenDatasetWith opens and returns a named Dataset with a user-defined PropList.
+// The returned dataset must be closed by the user when it is no longer needed.
+func (g *CommonFG) OpenDatasetWith(name string, dapl *PropList) (*Dataset, error) {
+	c_name := C.CString(name)
+	defer C.free(unsafe.Pointer(c_name))
+
+	hid := C.H5Dopen2(g.id, c_name, dapl.id)
+	if err := checkID(hid); err != nil {
+		return nil, err
+	}
+	return newDataset(hid, nil), nil
+}
+
 // NumObjects returns the number of objects in the Group.
 func (g *CommonFG) NumObjects() (uint, error) {
 	var info C.H5G_info_t

h5p_proplist.go

@@ -9,6 +9,7 @@ package hdf5
 // #include <string.h>
 // static inline hid_t _go_hdf5_H5P_DEFAULT() { return H5P_DEFAULT; }
 // static inline hid_t _go_hdf5_H5P_DATASET_CREATE() { return H5P_DATASET_CREATE; }
+// static inline hid_t _go_hdf5_H5P_DATASET_ACCESS() { return H5P_DATASET_ACCESS; }
 import "C"
 
 import (
@@ -32,6 +33,7 @@ type PropList struct {
 var (
 	P_DEFAULT        *PropList = newPropList(C._go_hdf5_H5P_DEFAULT())
 	P_DATASET_CREATE PropType  = PropType(C._go_hdf5_H5P_DATASET_CREATE()) // Properties for dataset creation
+	P_DATASET_ACCESS PropType  = PropType(C._go_hdf5_H5P_DATASET_ACCESS()) // Properties for dataset access
 )
 
 func newPropList(id C.hid_t) *PropList {
@@ -95,6 +97,25 @@ func (p *PropList) SetDeflate(level int) error {
 	return h5err(C.H5Pset_deflate(C.hid_t(p.id), C.uint(level)))
 }
 
+// SetChunkCache sets the raw data chunk cache parameters.
+// To reset them as default, use `D_CHUNK_CACHE_NSLOTS_DEFAULT`, `D_CHUNK_CACHE_NBYTES_DEFAULT` and `D_CHUNK_CACHE_W0_DEFAULT`.
+// https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetChunkCache
+func (p *PropList) SetChunkCache(nslots, nbytes int, w0 float64) error {
+	return h5err(C.H5Pset_chunk_cache(C.hid_t(p.id), C.size_t(nslots), C.size_t(nbytes), C.double(w0)))
+}
+
+// GetChunkCache retrieves the number of chunk slots in the raw data chunk cache hash table.
+// https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-GetChunkCache
+func (p *PropList) GetChunkCache() (nslots, nbytes int, w0 float64, err error) {
+	var (
+		c_nslots C.size_t
+		c_nbytes C.size_t
+		c_w0     C.double
+	)
+	err = h5err(C.H5Pget_chunk_cache(C.hid_t(p.id), &c_nslots, &c_nbytes, &c_w0))
+	return int(c_nslots), int(c_nbytes), float64(c_w0), err
+}
+
 func h5pclose(id C.hid_t) C.herr_t {
 	return C.H5Pclose(id)
 }

h5p_proplist_test.go

@@ -27,32 +27,32 @@ func TestChunk(t *testing.T) {
 	)
 	defer os.Remove(fn)
 
-	dclp, err := NewPropList(P_DATASET_CREATE)
+	dcpl, err := NewPropList(P_DATASET_CREATE)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer dclp.Close()
-	err = dclp.SetChunk(cdims)
+	defer dcpl.Close()
+	err = dcpl.SetChunk(cdims)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	cdims_, err := dclp.GetChunk(len(cdims))
+	cdimsChunk, err := dcpl.GetChunk(len(cdims))
 	if err != nil {
 		t.Fatal(err)
 	}
-	for i, cdim := range cdims_ {
+	for i, cdim := range cdimsChunk {
 		if cdim != cdims[i] {
 			t.Fatalf("chunked dimensions mismatch: %d != %d", cdims[i], cdim)
 		}
 	}
 
-	data0, err := save(fn, dsn, dims, dclp)
+	data0, err := save(fn, dsn, dims, dcpl)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	data1, err := load(fn, dsn)
+	data1, err := load(fn, dsn, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -73,26 +73,26 @@ func TestDeflate(t *testing.T) {
 	)
 	defer os.Remove(fn)
 
-	dclp, err := NewPropList(P_DATASET_CREATE)
+	dcpl, err := NewPropList(P_DATASET_CREATE)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer dclp.Close()
-	err = dclp.SetChunk(cdims)
+	defer dcpl.Close()
+	err = dcpl.SetChunk(cdims)
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = dclp.SetDeflate(DefaultCompression)
+	err = dcpl.SetDeflate(DefaultCompression)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	data0, err := save(fn, dsn, dims, dclp)
+	data0, err := save(fn, dsn, dims, dcpl)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	data1, err := load(fn, dsn)
+	data1, err := load(fn, dsn, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -102,7 +102,78 @@ func TestDeflate(t *testing.T) {
 	}
 }
 
-func save(fn, dsn string, dims []uint, dclp *PropList) ([]float64, error) {
+func TestChunkCache(t *testing.T) {
+	DisplayErrors(true)
+	defer DisplayErrors(false)
+	var (
+		fn    = "test_chunk_cache.h5"
+		dsn   = "dset_chunk_cache"
+		dims  = []uint{1000, 1000}
+		cdims = []uint{100, 100}
+	)
+	defer os.Remove(fn)
+
+	dcpl, err := NewPropList(P_DATASET_CREATE)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dcpl.Close()
+	err = dcpl.SetChunk(cdims)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cdimsChunk, err := dcpl.GetChunk(len(cdims))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i, cdim := range cdimsChunk {
+		if cdim != cdims[i] {
+			t.Fatalf("chunked dimensions mismatch: %d != %d", cdims[i], cdim)
+		}
+	}
+
+	data0, err := save(fn, dsn, dims, dcpl)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dapl, err := NewPropList(P_DATASET_ACCESS)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dapl.Close()
+
+	nslots, nbytes, w0, err := dapl.GetChunkCache()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	nslotsNew, nbytesNew, w0New := nslots*4, nbytes*2, w0/3
+	if err := dapl.SetChunkCache(nslotsNew, nbytesNew, w0New); err != nil {
+		t.Fatal(err)
+	}
+	if err := checkChunkCache(nslotsNew, nbytesNew, w0New, dapl); err != nil {
+		t.Fatal(err)
+	}
+
+	data1, err := load(fn, dsn, dapl)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := compare(data0, data1); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := dapl.SetChunkCache(D_CHUNK_CACHE_NSLOTS_DEFAULT, D_CHUNK_CACHE_NBYTES_DEFAULT, D_CHUNK_CACHE_W0_DEFAULT); err != nil {
+		t.Fatal(err)
+	}
+	if err := checkChunkCache(nslots, nbytes, w0, dapl); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func save(fn, dsn string, dims []uint, dcpl *PropList) ([]float64, error) {
 	f, err := CreateFile(fn, F_ACC_TRUNC)
 	if err != nil {
 		return nil, err
@@ -114,7 +185,7 @@ func save(fn, dsn string, dims []uint, dclp *PropList) ([]float64, error) {
 		return nil, err
 	}
 
-	dset, err := f.CreateDatasetWith(dsn, T_NATIVE_DOUBLE, dspace, dclp)
+	dset, err := f.CreateDatasetWith(dsn, T_NATIVE_DOUBLE, dspace, dcpl)
 	if err != nil {
 		return nil, err
 	}
@@ -132,14 +203,19 @@ func save(fn, dsn string, dims []uint, dclp *PropList) ([]float64, error) {
 	return data, nil
 }
 
-func load(fn, dsn string) ([]float64, error) {
+func load(fn, dsn string, dapl *PropList) ([]float64, error) {
 	f, err := OpenFile(fn, F_ACC_RDONLY)
 	if err != nil {
 		return nil, err
 	}
 	defer f.Close()
 
-	dset, _ := f.OpenDataset(dsn)
+	var dset *Dataset
+	if dapl == nil {
+		dset, err = f.OpenDataset(dsn)
+	} else {
+		dset, err = f.OpenDatasetWith(dsn, dapl)
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -168,3 +244,21 @@ func compare(ds0, ds1 []float64) error {
 	}
 	return nil
 }
+
+func checkChunkCache(nslots, nbytes int, w0 float64, dapl *PropList) error {
+	nslotsCache, nbytesCache, w0Cache, err := dapl.GetChunkCache()
+	if err != nil {
+		return err
+	}
+
+	if nslotsCache != nslots {
+		return fmt.Errorf("`nslots` mismatch: %d != %d", nslots, nslotsCache)
+	}
+	if nbytesCache != nbytes {
+		return fmt.Errorf("`nbytes` mismatch: %d != %d", nbytes, nbytesCache)
+	}
+	if math.Abs(w0Cache-w0) > 1e-5 {
+		return fmt.Errorf("`w0` mismatch: %.6f != %.6f", w0, w0Cache)
+	}
+	return nil
+}