forked from Unidata/netcdf-c
-
Notifications
You must be signed in to change notification settings - Fork 0
/
dfilterx.c
405 lines (343 loc) · 11 KB
/
dfilterx.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef _MSC_VER
#include <io.h>
#endif
#include "netcdf.h"
#include "netcdf_filter.h"
#include "ncdispatch.h"
#include "nc4internal.h"
#include "ncfilter.h"
#ifdef USE_HDF5
#include "hdf5internal.h"
#endif
/*
Unified filter related code
*/
#ifndef H5Z_FILTER_SZIP
/** ID of HDF SZIP filter. */
#define H5Z_FILTER_SZIP 4
#endif
/*Mnemonic*/
#define USENAME 1
#define LPAREN '('
#define RPAREN ')'
#define LBRACK '['
#define RBRACK ']'
#define NUMCHAR "0123456789"
#define NAMECHAR1 "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
#define NAMECHARN (NAMECHAR1 NUMCHAR "_-")
const struct FilterName {
const char* name; /* name or alias as assigned by HDF group*/
unsigned int id; /* id as assigned by HDF group*/
} known_filters[] = {
{"zip", 2}, /* Standard zlib compression */
{"zlib", 2}, /* alias */
{"deflate", 2}, /* alias */
{"szip", 4}, /* Standard szip compression */
{"bzip2", 307}, /* BZIP2 lossless compression used by PyTables */
{"lzf", 32000}, /* LZF lossless compression used by H5Py project */
{"blosc", 32001}, /* Blosc lossless compression used by PyTables */
{"mafisc", 32002}, /* Modified LZMA compression filter, MAFISC (Multidimensional Adaptive Filtering Improved Scientific data Compression) */
{"snappy", 32003}, /* Snappy lossless compression. */
{"lz4", 32004}, /* LZ4 fast lossless compression algorithm */
{"apax", 32005}, /* Samplify's APAX Numerical Encoding Technology */
{"cbf", 32006}, /* All imgCIF/CBF compressions and decompressions, including Canonical, Packed, Packed Vesrsion 2, Byte Offset and Nibble Offset. */
{"jpeg-xr", 32007}, /* Enables images to be compressed/decompressed with JPEG-XR compression */
{"bitshuffle", 32008}, /* Extreme version of shuffle filter that shuffles data at bit level instead of byte level. */
{"spdp", 32009}, /* SPDP fast lossless compression algorithm for single- and double-precision floating-point data. */
{"lpc-rice", 32010}, /* LPC-Rice multi-threaded lossless compression */
{"ccsds-123", 32011}, /* ESA CCSDS-123 multi-threaded compression filter */
{"jpeg-ls", 32012}, /* CharLS JPEG-LS multi-threaded compression filter */
{"zfp", 32013}, /* Rate, accuracy or precision bounded compression for floating-point arrays */
{"fpzip", 32014}, /* Fast and Efficient Lossy or Lossless Compressor for Floating-Point Data */
{"zstandard", 32015}, /* Real-time compression algorithm with wide range of compression / speed trade-off and fast decoder */
{"b3d", 32016}, /* GPU based image compression method developed for light-microscopy applications */
{"sz", 32017}, /* An error-bounded lossy compressor for scientific floating-point data */
{"fcidecomp", 32018}, /* EUMETSAT CharLS compression filter for use with netCDF */
{"user-defined", 32768}, /* First user-defined filter */
{NULL,0}
};
/**************************************************/
/*Forward*/
static unsigned int NC_filterx_lookup(const char* filtername);
static const char* NC_filterx_toname(unsigned int id);
static int NC_filterx_transferstringvec(size_t n, char** vec, char** copy);
/**************************************************/
/* Per-variable filters extended string-based */
/**
Find the set of filters (if any) associated with a variable.
\param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
\param varid Variable ID
\param nfilters return no. of filters
\param ids return the filter ids (caller allocates)
\returns ::NC_NOERR No error.
\returns ::NC_ENOTNC4 Not a netCDF-4 file.
\returns ::NC_EBADID Bad ncid.
\returns ::NC_ENOTVAR Invalid variable ID.
\returns ::NC_EINVAL Invalid arguments
\ingroup variables
\author Dennis Heimbigner
*/
EXTERNL int
nc_inq_var_filterx_ids(int ncid, int varid, size_t* nfiltersp, char** ids)
{
NC* ncp;
int stat = NC_NOERR;
NC_FILTERX_OBJ ncids;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_inq_var_filterids);
memset(&ncids,0,sizeof(ncids));
ncids.usort = NC_FILTER_UNION_IDS;
ncids.u.ids.nfilters = 0;
ncids.u.ids.filterids = NULL;
if((stat = ncp->dispatch->filter_actions(ncid,varid, NCFILTER_FILTERIDS, &ncids))) goto done;
if(nfiltersp) *nfiltersp = ncids.u.ids.nfilters;
if(ids) {
if((stat = NC_filterx_transferstringvec(ncids.u.ids.nfilters, ncids.u.ids.filterids, ids))) goto done;
}
done:
NC_filterx_freestringvec(ncids.u.ids.nfilters,ncids.u.ids.filterids);
return stat;
}
/**
Find the the param info about filter (if any)
associated with a variable and with specified id.
This is a wrapper for nc_inq_var_all().
\param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
\param varid Variable ID
\param id The filter id of interest
\param formatp (Out) Storage for the filter format
\param nparamsp (Out) Storage which will get the number of parameters to the filter
\param params (Out) Storage which will get associated parameters.
Note: the caller must allocate and free.
\returns ::NC_NOERR No error.
\returns ::NC_ENOTNC4 Not a netCDF-4 file.
\returns ::NC_EBADID Bad ncid.
\returns ::NC_ENOTVAR Invalid variable ID.
\returns ::NC_ENOFILTER No filter defined.
\ingroup variables
\author Dennis Heimbigner
*/
EXTERNL int
nc_inq_var_filterx_info(int ncid, int varid, const char* id, size_t* nparamsp, char** params)
{
NC* ncp;
int stat = NC_NOERR;
NC_FILTERX_OBJ spec;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_inq_var_filter_info);
memset(&spec,0,sizeof(spec));
spec.usort = NC_FILTER_UNION_SPEC;
spec.u.spec.filterid = (char*)id;
spec.u.spec.params = NULL;
if((stat = ncp->dispatch->filter_actions(ncid,varid,NCFILTER_INFO,&spec))) goto done;
if(nparamsp) *nparamsp = spec.u.spec.nparams;
if(params) {
if((stat = NC_filterx_transferstringvec(spec.u.spec.nparams, spec.u.spec.params, params))) goto done;
}
done:
NC_filterx_freestringvec(spec.u.spec.nparams,spec.u.spec.params);
return stat;
}
/**
Define a new variable filter.
Only variables with chunked storage can use filters.
@param ncid File and group ID.
@param varid Variable ID.
@param id Filter ID.
@param nparams Number of filter parameters.
@param parms Filter parameters.
@return ::NC_NOERR No error.
@return ::NC_EINVAL Variable must be chunked.
@return ::NC_EBADID Bad ID.
@author Dennis Heimbigner
*/
EXTERNL int
nc_def_var_filterx(int ncid, int varid, const char* id, size_t nparams, const char** params)
{
NC* ncp;
NC_FILTERX_OBJ spec;
int stat = NC_NOERR;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_def_var_filter);
memset(&spec,0,sizeof(spec));
spec.usort = NC_FILTER_UNION_SPEC;
spec.u.spec.filterid = (char*)id;
spec.u.spec.nparams = nparams;
spec.u.spec.params = (char**)params; /* discard const */
if((stat=ncp->dispatch->filter_actions(ncid,varid,NCFILTER_DEF,&spec))) goto done;
done:
return stat;
}
/**
Remove all filters with specified id from a variable
@param ncid File and group ID.
@param varid Variable ID.
@param id filter to remove
@return ::NC_NOERR No error.
@return ::NC_EBADID Bad ID.
@author Dennis Heimbigner
*/
EXTERNL int
nc_var_filterx_remove(int ncid, int varid, const char* id)
{
NC* ncp;
int stat = NC_NOERR;
NC_FILTERX_OBJ spec;
if((stat = NC_check_id(ncid,&ncp))) goto done;
TRACE(ncx_var_filter_remove);
memset(&spec,0,sizeof(spec));
spec.usort = NC_FILTER_UNION_SPEC;
spec.u.spec.filterid = (char*)id;
if((stat=ncp->dispatch->filter_actions(ncid,varid,NCFILTER_REMOVE,&spec))) goto done;
done:
return stat;
}
/**************************************************/
/* Utilities */
int
NC_cvtX2I_idlist(size_t n, const char** xidlist, unsigned int* ids)
{
int i,stat = NC_NOERR;
for(i=0;i<n;i++) {
if((stat = NC_cvtX2I_id(xidlist[i],&ids[i]))) break;
}
return stat;
}
int
NC_cvtX2I_params(size_t nparams, const char** xparamslist, unsigned int* params)
{
int i,stat = NC_NOERR;
unsigned int id;
for(i=0;i<nparams;i++) {
/* See if this param looks like an unsigned int */
if(sscanf(xparamslist[i],"%u",&id) == 1) {
params[i] = id;
}
}
return stat;
}
int
NC_cvtI2X_idlist(int n, const unsigned int* ids, char** xid)
{
int i,stat = NC_NOERR;
char sid[1024];
/* For now, do not attempt a name conversion */
for(i=0;i<n;i++) {
snprintf(sid,sizeof(sid),"%u",ids[i]);
if((xid[i] = strdup(sid)) == NULL)
{stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
int
NC_cvtI2X_params(int n, const unsigned int* ids, char** params)
{
int i,stat = NC_NOERR;
char sid[1024];
for(i=0;i<n;i++) {
snprintf(sid,sizeof(sid),"%u",ids[i]);
if((params[i] = strdup(sid))==NULL)
{stat = NC_ENOMEM; goto done;}
}
done:
return stat;
}
/* Convert an xid; allow a name */
int
NC_cvtX2I_id(const char* xid, unsigned int* idp)
{
unsigned int id;
/* See if this id looks like an unsigned int */
if(sscanf(xid,"%u",&id) != 1) {
id = NC_filterx_lookup(xid);
}
if(idp) *idp = id;
return (id > 0 ? NC_NOERR : NC_EINVAL);
}
/* Convert an int id to a string; optional name conversion */
int
NC_cvtI2X_id(unsigned int id, char** xidp, int usename)
{
char xid[NC_MAX_NAME];
snprintf(xid,sizeof(xid),"%u",id);
if(usename) {/* See if this id has a name */
const char* name = NC_filterx_toname(id);
if(name != NULL) {xid[0] = '\0'; strlcat(xid,name,sizeof(xid));}
}
if(xidp) {
if((*xidp = strdup(xid))==NULL) return NC_ENOMEM;
}
return NC_NOERR;
}
static unsigned int
NC_filterx_lookup(const char* filtername)
{
const struct FilterName* p = known_filters;
/* Try a name lookup */
for(;p->name;p++) {
if(strcasecmp(p->name,filtername)==0)
return p->id;
}
return 0; /* no match */
}
static const char*
NC_filterx_toname(unsigned int id)
{
const struct FilterName* p = known_filters;
for(;p->name;p++) {
if(p->id == id)
return p->name;
}
return NULL; /* no match */
}
void
NC_filterx_freestringvec(size_t n, char** vec)
{
int i;
if(vec != NULL) {
for(i=0;i<n;i++) {
if(vec[i]) free(vec[i]);
}
free(vec);
}
}
static int
NC_filterx_transferstringvec(size_t n, char** vec, char** copy)
{
int i, stat = NC_NOERR;
for(i=0;i<n;i++) {
copy[i] = vec[i];
vec[i] = NULL;
}
return stat;
}
int
NC_filterx_copy(size_t n, const char** vec, char*** copyp)
{
char** copy = NULL;
int i, stat = NC_NOERR;
if((copy = calloc(sizeof(char*),n+1))==NULL)
{stat = NC_ENOMEM; goto done;}
for(i=0;i<n;i++) {
char* s = nulldup(vec[i]);
if(s == NULL) {stat = NC_ENOMEM; goto done;}
copy[i] = s;
}
if(copyp) {*copyp = copy; copy = NULL;}
done:
if(copy) NC_filterx_freestringvec(n,copy);
return stat;
}