Merge pull request #1375 from Unidata/addfilter.dmh

Allow programmatic definition of filters

docs/filters.md

@@ -508,13 +508,93 @@ for use by client programs and by filter implementations.
 Examples of the use of these functions can be seen in the test program
 *nc_test4/tst_filterparser.c*.
 
+Appendix B. Programmatic Filter Definition {#filters_programmatic}
+==========
+
+HDF5 provides an API [6] to allow for the programmatic definition
+of filters -- as opposed to using the HDF5_PLUGIN_PATH environment variable.
+The idea is that instead of using dynamic shared libraries, the filter code
+is compiled into the application and the relevant information
+(namely an instance of *H5Z_class2_t*) is passed to the HDF5 library API.
+Because it is anticipated that in the future, other plugin formats
+will be used, this netcdf-c API is deliberately more general than
+strictly required by HDF5.
+
+## API Concepts
+
+Three concepts are used in this API.
+
+1. Format - this is an integer defining the format of the plugin.
+   Currently, only *NC_FILTER_FORMAT_HDF5* is defined and corresponds
+   to the existing HDF5 plugin format.
+2. ID - this is an integer that is a unique identifier for the filter.
+   This value is interpreted in the context of the format, so the same
+   id might be assigned to different filters if the format is different.
+3. The structure *NC_FILTER_INFO* that provides generic information
+   to the API and has a placeholder for format-specific information.
+
+        typedef struct NC_FILTER_INFO {
+          int version; /* Of this structure */
+          int format; /* Controls actual type of this structure */
+          int id;     /* Must be unique WRT format */
+          void* info; /* The filter info as defined by the format. */
+        } NC_FILTER_INFO;
+    When the format is the value NC_FILTER_FORMAT_HDF5,
+    then the info field is a pointer to an instance of
+    H5Z_class2_t as define in H5Zpublic.h.
+    The use of void* is, of course, to allow for passing arbitrary objects.
+
+### NetCDF API
+
+The following function signatures are provided (see *netcdf_filter.h*).
+
+1. Register a filter
+
+        int nc_filter_register(NC_FILTER_INFO* filter_info);
+    Register a filter whose format and ID are specified in the 'filter_info'
+    argument.
+
+2. Unregister a filter
+
+        int nc_filter_unregister(int format, int id);
+    Unregister the filter specified by the id. Note that only
+    filters registered using 'nc_filter_register' can be unregistered.
+
+3. Inquire about a filter
+
+        int nc_filter_inq(int format, int id, NC_FILTER_INFO* filter_info);
+    Unregister the filter specified by the id. Note that only
+    filters registered using 'nc_filter_register' can be inquired.
+    The 'filter_info' is filled with a copy of the original argument to
+    'nc_filter_register'.
+
+### Example
+
+    static const H5Z_class2_t H5Z_REG[1] = {
+        ...
+    };
+    ...
+    NC_FILTER_INFO info;
+    ...
+    info.version = NC_FILTER_INFO_VERSION;
+    info.format = NC_FILTER_FORMAT_HDF5;
+    info.id = FILTER_ID;
+    info.info = (void*)&H5Z_REG[0];
+    stat = nc_filter_register(&info);
+    ...
+    memset(&info,0,sizeof(NC_FILTER_INFO));
+    stat = nc_filter_inq(NC_FILTER_FORMAT_HDF5, FILTER_ID, &info);
+    ...
+    stat = nc_filter_unregister(NC_FILTER_FORMAT_HDF5, FILTER_ID);
+
 # References {#filters_References}
 
 1. https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf
 2. https://support.hdfgroup.org/HDF5/doc/TechNotes/TechNote-HDF5-CompressionTroubleshooting.pdf
 3. https://portal.hdfgroup.org/display/support/Contributions#Contributions-filters
 4. https://support.hdfgroup.org/services/contributions.html#filters
 5. https://support.hdfgroup.org/HDF5/doc/RM/RM_H5.html
+6. https://confluence.hdfgroup.org/display/HDF5/Filters
 
 # Point of Contact
 

include/hdf5internal.h

@@ -18,6 +18,7 @@
 #include "ncdimscale.h"
 #include "nc4dispatch.h"
 #include "hdf5dispatch.h"
+#include "netcdf_filter.h"
 
 #define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10)
 
@@ -51,6 +52,11 @@
 /** This is the name of the name HDF5 dimension scale attribute. */
 #define HDF5_DIMSCALE_NAME_ATT_NAME "NAME"
 
+/** Define Filter API Operations */
+#define FILTER_REG   1
+#define FILTER_UNREG 2
+#define FILTER_INQ   3
+
 /** Struct to hold HDF5-specific info for the file. */
 typedef struct NC_HDF5_FILE_INFO {
    hid_t hdfid;
@@ -163,9 +169,13 @@ int nc4_hdf5_find_grp_h5_var(int ncid, int varid, NC_FILE_INFO_T **h5,
 /* Perform lazy read of the rest of the metadata for a var. */
 int nc4_get_var_meta(NC_VAR_INFO_T *var);
 
+
+/* Define Filter API Function */
+int nc4_filter_action(int action, int formatx, int id, NC_FILTER_INFO* info);
 /* Support functions for provenance info (defined in nc4hdf.c) */
 extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hdf.c*/
 extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/
 extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/
 
+
 #endif /* _HDF5INTERNAL_ */

include/netcdf_filter.h

@@ -11,6 +11,25 @@
 #define H5Z_FILTER_SZIP 4
 #endif
 
+/* Define the known filter formats */
+#define NC_FILTER_FORMAT_HDF5 1 /* Use the H5Z_class2_t format */
+
+/* Note that this structure can be extended
+   in the usual C way if the first field of the extended
+   struct is of type NC_FILTER_INFO
+*/
+typedef struct NC_FILTER_INFO {
+    int version; /* Of this structure */
+#      define NC_FILTER_INFO_VERSION 1
+    int format; /* Controls actual type of this structure */
+    int id;     /* Must be unique WRT format */
+    void* info; /* The filter info as defined by the format.
+                   For format == NC_FILTER_FORMAT_HDF5,
+                   this must conform to H5Z_class2_t in H5Zpublic.h;
+                   Defined as void* to avoid specifics.
+                */
+} NC_FILTER_INFO;
+
 #if defined(__cplusplus)
 extern "C" {
 #endif
@@ -20,6 +39,11 @@ EXTERNL int NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* npar
 
 EXTERNL void NC_filterfix8(unsigned char* mem, int decode);
 
+/* Support direct user defined filters */
+EXTERNL int nc_filter_register(NC_FILTER_INFO* filter_info);
+EXTERNL int nc_filter_unregister(int format, int id);
+EXTERNL int nc_filter_inq(int format, int id, NC_FILTER_INFO* filter_info);
+
 #if defined(__cplusplus)
 }
 #endif

libdispatch/dfilter.c

@@ -14,6 +14,10 @@
 #include "netcdf.h"
 #include "netcdf_filter.h"
 
+#ifdef USE_NETCDF4
+#include "hdf5internal.h"
+#endif
+
 /*
 Common utilities related to filters.
 */
@@ -262,3 +266,62 @@ NC_filterfix8(unsigned char* mem, int decode)
     /* No action is necessary */
 #endif	    
 }
+
+/**************************************************/
+/* Support direct user defined filters */
+
+EXTERNL int
+nc_filter_register(NC_FILTER_INFO* filter)
+{
+    int stat = NC_NOERR;
+    if(filter == NULL)
+	return NC_EINVAL;
+    switch (filter->format) {
+    case NC_FILTER_FORMAT_HDF5:
+#ifdef USE_NETCDF4
+	stat = nc4_filter_action(FILTER_REG, filter->format, filter->id, filter);
+#else
+        stat = NC_ENOTBUILT;
+#endif
+	break;
+    default:
+	stat = NC_EINVAL;
+    }
+    return stat;
+}
+
+EXTERNL int
+nc_filter_unregister(int fformat, int id)
+{
+    int stat = NC_NOERR;
+    switch (fformat) {
+    case NC_FILTER_FORMAT_HDF5:
+#ifdef USE_NETCDF4
+	stat = nc4_filter_action(FILTER_UNREG, fformat, id, NULL);
+#else
+        stat = NC_ENOTBUILT;
+#endif
+	break;
+    default:
+	stat = NC_EINVAL;
+    }
+    return stat;
+}
+
+EXTERNL int
+nc_filter_inq(int fformat, int id, NC_FILTER_INFO* filter_info)
+{
+    int stat = NC_NOERR;
+    switch (fformat) {
+    case NC_FILTER_FORMAT_HDF5:
+#ifdef USE_NETCDF4
+	stat = nc4_filter_action(FILTER_INQ, fformat, id, filter_info);
+#else
+        stat = NC_ENOTBUILT;
+#endif
+	break;
+    default:
+	stat = NC_EINVAL;
+    }
+    return stat;
+}

libhdf5/nc4hdf.c

@@ -25,6 +25,11 @@
 
 #define NC_HDF5_MAX_NAME 1024 /**< @internal Max size of HDF5 name. */
 
+/* WARNING: GLOBAL VARIABLE */
+
+/* Define list of registered filters */
+static NClist* filters = NULL;
+
 /**
  * @internal Flag attributes in a linked list as dirty.
  *
@@ -2638,3 +2643,122 @@ NC4_walk(hid_t gid, int* countp)
     }
     return ncstat;
 }
+
+
+/**************************************************/
+/* Filter registration support */
+
+static int
+filterlookup(int id)
+{
+    int i;
+    if(filters == NULL)
+	filters = nclistnew();
+    for(i=0;i<nclistlength(filters);i++) {
+	NC_FILTER_INFO* x = nclistget(filters,i);
+	if(x != NULL && x->id == id) return i; /* return position */
+    }
+    return -1;
+}
+
+static void
+reclaiminfo(NC_FILTER_INFO* info)
+{
+    if(info != NULL)
+        nullfree(info->info);
+    nullfree(info);
+}
+
+static int
+filterremove(int pos)
+{
+    NC_FILTER_INFO* info = NULL;
+    if(filters == NULL)
+	filters = nclistnew();
+    if(pos < 0 || pos >= nclistlength(filters))
+	return NC_EINVAL;
+    info = nclistget(filters,pos);
+    reclaiminfo(info);
+    nclistremove(filters,pos);
+    return NC_NOERR;
+}
+
+static NC_FILTER_INFO*
+dupfilterinfo(NC_FILTER_INFO* info)
+{
+    NC_FILTER_INFO* dup = NULL;
+    if(info == NULL) goto fail;
+    if(info->info == NULL) goto fail;
+    if((dup = calloc(1,sizeof(NC_FILTER_INFO))) == NULL) goto fail;
+    *dup = *info;
+    if((dup->info = calloc(1,sizeof(H5Z_class2_t))) == NULL) goto fail;
+    {
+        H5Z_class2_t* h5dup = (H5Z_class2_t*)dup->info;
+        H5Z_class2_t* h5info = (H5Z_class2_t*)info->info;
+        *h5dup = *h5info;
+    }
+    return dup;
+fail:
+    reclaiminfo(dup);
+    return NULL;
+}
+
+int
+nc4_filter_action(int op, int format, int id, NC_FILTER_INFO* info)
+{
+    int stat = NC_NOERR;
+    H5Z_class2_t* h5filterinfo = NULL;
+    herr_t herr;
+    int pos = -1;
+    NC_FILTER_INFO* dup = NULL;
+
+    if(format != NC_FILTER_FORMAT_HDF5)
+	{stat = NC_ENOTNC4; goto done;}
+
+    switch (op) {
+    case FILTER_REG: /* Ignore id argument */
+	if(info == NULL || info->info == NULL)
+	    {stat = NC_EINVAL; goto done;}
+	if(info->version != NC_FILTER_INFO_VERSION
+	   || info->format != NC_FILTER_FORMAT_HDF5)
+	    {stat = NC_ENOTNC4; goto done;}
+        h5filterinfo = info->info;
+        /* Another sanity check */
+        if(info->id != h5filterinfo->id)
+	    {stat = NC_EINVAL; goto done;}
+	/* See if this filter is already defined */
+	if((pos = filterlookup(id)) >= 0)
+	    {stat = NC_ENAMEINUSE; goto done;} /* Already defined */
+	if((herr = H5Zregister(h5filterinfo)) < 0)
+	    {stat = NC_EFILTER; goto done;}
+	/* Save a copy of the passed in info */
+	if((dup = dupfilterinfo(info)) == NULL)
+	    {stat = NC_ENOMEM; goto done;}		
+	nclistpush(filters,dup);	
+	break;
+    case FILTER_UNREG:
+	if(id <= 0)
+	    {stat = NC_ENOTNC4; goto done;}
+	/* See if this filter is already defined */
+	if((pos = filterlookup(id)) < 0)
+	    {stat = NC_EFILTER; goto done;} /* Not defined */
+	if((herr = H5Zunregister(id)) < 0)
+	    {stat = NC_EFILTER; goto done;}
+	if((stat=filterremove(pos))) goto done;
+	break;
+    case FILTER_INQ:
+	if(id <= 0)
+	    {stat = NC_ENOTNC4; goto done;}
+	/* Look up the id in our local table */
+	if((pos = filterlookup(id)) < 0)
+	    {stat = NC_EFILTER; goto done;} /* Not defined */
+	if(info != NULL) {
+	    *info = *((NC_FILTER_INFO*)nclistget(filters,pos));
+	}
+	break;
+    default:
+	{stat = NC_EINTERNAL; goto done;}	
+    }
+done:
+    return stat;
+} 

nc_test4/CMakeLists.txt

@@ -32,8 +32,9 @@ IF(BUILD_UTILITIES)
 IF(ENABLE_FILTER_TESTING)
   build_bin_test(test_filter)
   build_bin_test(test_filter_misc)
+  build_bin_test(test_filter_reg)
   ADD_SH_TEST(nc_test4 tst_filter)
-  SET(NC4_TESTS ${NC4_TESTS} tst_filterparser)
+  SET(NC4_TESTS ${NC4_TESTS} tst_filterparser test_filter_reg)
 ENDIF(ENABLE_FILTER_TESTING)
 
 ENDIF(BUILD_UTILITIES)