From c23287b44b5de2071ff031928361200ff21b9474 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Mon, 27 Jun 2022 08:59:11 -0600 Subject: [PATCH 01/13] use inq_filter_avail to check if filters are usable --- include/netCDF4.pxi | 6 ++++++ src/netCDF4/_netCDF4.pyx | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/include/netCDF4.pxi b/include/netCDF4.pxi index 8be538234..ba0190551 100644 --- a/include/netCDF4.pxi +++ b/include/netCDF4.pxi @@ -698,9 +698,13 @@ IF HAS_QUANTIZATION_SUPPORT: NC_QUANTIZE_BITROUND int nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd) int nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) nogil + cdef extern from "netcdf_filter.h": + int nc_inq_filter_avail(int ncid, unsigned filterid); IF HAS_SZIP_SUPPORT: cdef extern from "netcdf.h": + cdef enum: + H5Z_FILTER_SZIP int nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd) int nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) nogil int nc_def_var_szip(int ncid, int varid, int options_mask, int pixels_per_bloc) @@ -723,6 +727,8 @@ IF HAS_BZIP2_SUPPORT: IF HAS_BLOSC_SUPPORT: cdef extern from "netcdf_filter.h": + cdef enum: + H5Z_FILTER_BLOSC int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle) int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep) diff --git a/src/netCDF4/_netCDF4.pyx b/src/netCDF4/_netCDF4.pyx index 8a006f457..c782456a5 100644 --- a/src/netCDF4/_netCDF4.pyx +++ b/src/netCDF4/_netCDF4.pyx @@ -1446,6 +1446,18 @@ _nctonptype[NC_CHAR]='S1' # internal C functions. +def has_blasc_filter(): + """check to see if blosc compression filter is available""" + cdef int ierr + IF HAS_BLOSC_SUPPORT: + ierr = nc_inq_filter_avail(0, H5Z_FILTER_BLOSC) + if ierr: + return False + else: + return True + ELSE: + return False + cdef _get_att_names(int grpid, int varid): # Private function to get all the attribute names in a group cdef int ierr, numatts, n From 0142a367ee34078c66aa23d41544a2c5c6fc0148 Mon Sep 17 00:00:00 2001 From: jswhit Date: Mon, 27 Jun 2022 09:48:25 -0600 Subject: [PATCH 02/13] add Dataset methods to check for available of blosc,zstd,bzip2 and szip --- include/netCDF4.pxi | 2 +- src/netCDF4/_netCDF4.pyx | 64 ++++++++++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/include/netCDF4.pxi b/include/netCDF4.pxi index ba0190551..6c27cd246 100644 --- a/include/netCDF4.pxi +++ b/include/netCDF4.pxi @@ -713,7 +713,7 @@ IF HAS_SZIP_SUPPORT: IF HAS_ZSTANDARD_SUPPORT: cdef extern from "netcdf_filter.h": cdef enum: - H5Z_FILTER_ZSTANDARD + H5Z_FILTER_ZSTD int nc_def_var_zstandard(int ncid, int varid, int level) int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp) int nc_inq_filter_avail(int ncid, unsigned id) diff --git a/src/netCDF4/_netCDF4.pyx b/src/netCDF4/_netCDF4.pyx index c782456a5..07c7a0d29 100644 --- a/src/netCDF4/_netCDF4.pyx +++ b/src/netCDF4/_netCDF4.pyx @@ -1446,18 +1446,6 @@ _nctonptype[NC_CHAR]='S1' # internal C functions. -def has_blasc_filter(): - """check to see if blosc compression filter is available""" - cdef int ierr - IF HAS_BLOSC_SUPPORT: - ierr = nc_inq_filter_avail(0, H5Z_FILTER_BLOSC) - if ierr: - return False - else: - return True - ELSE: - return False - cdef _get_att_names(int grpid, int varid): # Private function to get all the attribute names in a group cdef int ierr, numatts, n @@ -3373,6 +3361,58 @@ to be installed and in `$PATH`. f = open(outfile,'w') f.write(result.stdout) f.close() + def has_blosc_filter(self): + """ +**`has_blosc_filter(self)`** +returns True if blosc compression filter is available""" + cdef int ierr + IF HAS_BLOSC_SUPPORT: + ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_BLOSC) + if ierr: + return False + else: + return True + ELSE: + return False + def has_zstd_filter(self): + """ +**`has_zstd_filter(self)`** +returns True if zstd compression filter is available""" + cdef int ierr + IF HAS_ZSTANDARD_SUPPORT: + ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_ZSTD) + if ierr: + return False + else: + return True + ELSE: + return False + def has_bzip2_filter(self): + """ +**`has_bzip2_filter(self)`** +returns True if bzip2 compression filter is available""" + cdef int ierr + IF HAS_BZIP2_SUPPORT: + ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_BZIP2) + if ierr: + return False + else: + return True + ELSE: + return False + def has_szip_filter(self): + """ +**`has_szip_filter(self)`** +returns True if szip compression filter is available""" + cdef int ierr + IF HAS_SZIP_SUPPORT: + ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_SZIP) + if ierr: + return False + else: + return True + ELSE: + return False cdef class Group(Dataset): """ From 787e252d16e1eeda4df3c354bb11315b29713a7a Mon Sep 17 00:00:00 2001 From: jswhit Date: Mon, 27 Jun 2022 09:52:38 -0600 Subject: [PATCH 03/13] skip blosc tests if filter not available --- test/tst_compression_blosc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/tst_compression_blosc.py b/test/tst_compression_blosc.py index da31d3ce1..3f799c8b7 100644 --- a/test/tst_compression_blosc.py +++ b/test/tst_compression_blosc.py @@ -74,4 +74,9 @@ def runTest(self): f.close() if __name__ == '__main__': - unittest.main() + nc = Dataset(filename,'w') + if not nc.has_blosc_filter(): + sys.stdout.write('blosc filter not available, skipping tests ...\n') + else: + nc.close() + unittest.main() From f308cda7a1d51ebd72e60cf4dd078226e37b8cca Mon Sep 17 00:00:00 2001 From: jswhit Date: Mon, 27 Jun 2022 12:23:57 -0600 Subject: [PATCH 04/13] skip tests if filters not available --- test/tst_compression_bzip2.py | 7 ++++++- test/tst_compression_szip.py | 7 ++++++- test/tst_compression_zstd.py | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/test/tst_compression_bzip2.py b/test/tst_compression_bzip2.py index 89de4086c..732e18da3 100644 --- a/test/tst_compression_bzip2.py +++ b/test/tst_compression_bzip2.py @@ -49,4 +49,9 @@ def runTest(self): f.close() if __name__ == '__main__': - unittest.main() + nc = Dataset(filename1,'w') + if not nc.has_bzip2_filter(): + sys.stdout.write('bzip2 filter not available, skipping tests ...\n') + else: + nc.close() + unittest.main() diff --git a/test/tst_compression_szip.py b/test/tst_compression_szip.py index 874ed3821..bac2bce8e 100644 --- a/test/tst_compression_szip.py +++ b/test/tst_compression_szip.py @@ -39,4 +39,9 @@ def runTest(self): f.close() if __name__ == '__main__': - unittest.main() + nc = Dataset(filename,'w') + if not nc.has_szip_filter(): + sys.stdout.write('szip filter not available, skipping tests ...\n') + else: + nc.close() + unittest.main() diff --git a/test/tst_compression_zstd.py b/test/tst_compression_zstd.py index 2745e03ba..992928a44 100644 --- a/test/tst_compression_zstd.py +++ b/test/tst_compression_zstd.py @@ -49,4 +49,9 @@ def runTest(self): f.close() if __name__ == '__main__': - unittest.main() + nc = Dataset(filename1,'w') + if not nc.has_zstd_filter(): + sys.stdout.write('blosc filter not available, skipping tests ...\n') + else: + nc.close() + unittest.main() From 1010b05d389838acb7cda1ac7f708e4837b90d3c Mon Sep 17 00:00:00 2001 From: jswhit Date: Mon, 27 Jun 2022 12:28:10 -0600 Subject: [PATCH 05/13] don't install plugins to see if tests are skipped --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 11e218157..28e36a134 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,7 +61,7 @@ jobs: - name: Install netcdf4-python run: | export PATH=${NETCDF_DIR}/bin:${PATH} - export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir + #export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir python setup.py install - name: Test run: | From fe0cec0f0fa5b8509936855da5210609ea873848 Mon Sep 17 00:00:00 2001 From: jswhit Date: Mon, 27 Jun 2022 14:28:30 -0600 Subject: [PATCH 06/13] fix import --- test/tst_compression_blosc.py | 2 +- test/tst_compression_bzip2.py | 2 +- test/tst_compression_szip.py | 2 +- test/tst_compression_zstd.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/tst_compression_blosc.py b/test/tst_compression_blosc.py index 3f799c8b7..5b45c1857 100644 --- a/test/tst_compression_blosc.py +++ b/test/tst_compression_blosc.py @@ -1,7 +1,7 @@ from numpy.random.mtrand import uniform from netCDF4 import Dataset from numpy.testing import assert_almost_equal -import os, tempfile, unittest +import os, tempfile, unittest, sys ndim = 100000 iblosc_shuffle=2 diff --git a/test/tst_compression_bzip2.py b/test/tst_compression_bzip2.py index 732e18da3..22149b60e 100644 --- a/test/tst_compression_bzip2.py +++ b/test/tst_compression_bzip2.py @@ -1,7 +1,7 @@ from numpy.random.mtrand import uniform from netCDF4 import Dataset from numpy.testing import assert_almost_equal -import os, tempfile, unittest +import os, tempfile, unittest, sys ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name diff --git a/test/tst_compression_szip.py b/test/tst_compression_szip.py index bac2bce8e..3dc3cb8db 100644 --- a/test/tst_compression_szip.py +++ b/test/tst_compression_szip.py @@ -1,7 +1,7 @@ from numpy.random.mtrand import uniform from netCDF4 import Dataset from numpy.testing import assert_almost_equal -import os, tempfile, unittest +import os, tempfile, unittest, sys ndim = 100000 filename = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name diff --git a/test/tst_compression_zstd.py b/test/tst_compression_zstd.py index 992928a44..0e14436e6 100644 --- a/test/tst_compression_zstd.py +++ b/test/tst_compression_zstd.py @@ -1,7 +1,7 @@ from numpy.random.mtrand import uniform from netCDF4 import Dataset from numpy.testing import assert_almost_equal -import os, tempfile, unittest +import os, tempfile, unittest, sys ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name From 4052bde62fc5f438fc446f585a543772f5abcc3e Mon Sep 17 00:00:00 2001 From: jswhit Date: Mon, 27 Jun 2022 16:08:36 -0600 Subject: [PATCH 07/13] re-enable plugin install --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 28e36a134..11e218157 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,7 +61,7 @@ jobs: - name: Install netcdf4-python run: | export PATH=${NETCDF_DIR}/bin:${PATH} - #export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir + export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir python setup.py install - name: Test run: | From 5c58e70f8467e2c0c3a3c8fe4cc206ac6529b66d Mon Sep 17 00:00:00 2001 From: jswhit Date: Mon, 27 Jun 2022 16:56:51 -0600 Subject: [PATCH 08/13] update --- .github/workflows/build_master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index 00b3a52d4..4b1490bda 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -50,7 +50,7 @@ jobs: - name: Install netcdf4-python run: | export PATH=${NETCDF_DIR}/bin:${PATH} - export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir + #export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir python setup.py install - name: Test run: | From 3d6785188b9cb84b071c9c8002a2cb38db7abd31 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Mon, 27 Jun 2022 18:18:21 -0600 Subject: [PATCH 09/13] update --- .github/workflows/build_master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index 4b1490bda..00b3a52d4 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -50,7 +50,7 @@ jobs: - name: Install netcdf4-python run: | export PATH=${NETCDF_DIR}/bin:${PATH} - #export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir + export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir python setup.py install - name: Test run: | From ec63a60d946600ec360935e86ae2cf43073810df Mon Sep 17 00:00:00 2001 From: jswhit Date: Mon, 27 Jun 2022 18:30:40 -0600 Subject: [PATCH 10/13] update --- test/run_all.py | 16 ++++++++++------ test/tst_compression_zstd.py | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/test/run_all.py b/test/run_all.py index 1dfb5309a..f2ff97a18 100755 --- a/test/run_all.py +++ b/test/run_all.py @@ -1,5 +1,5 @@ -import glob, os, sys, unittest, struct -from netCDF4 import getlibversion,__hdf5libversion__,__netcdf4libversion__,__version__ +import glob, os, sys, unittest, struct, tempfile +from netCDF4 import getlibversion,__hdf5libversion__,__netcdf4libversion__,__version__, Dataset from netCDF4 import __has_cdf5_format__, __has_nc_inq_path__, __has_nc_create_mem__, \ __has_parallel4_support__, __has_pnetcdf_support__, \ __has_zstandard_support__, __has_bzip2_support__, \ @@ -26,18 +26,22 @@ if not __has_quantization_support__: test_files.remove('tst_compression_quant.py') sys.stdout.write('not running tst_compression_quant.py ...\n') -if not __has_zstandard_support__ or os.getenv('NO_PLUGINS'): +filename = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name +nc = Dataset(filename,'w') +if not __has_zstandard_support__ or os.getenv('NO_PLUGINS') or not nc.has_zstd_filter(): test_files.remove('tst_compression_zstd.py') sys.stdout.write('not running tst_compression_zstd.py ...\n') -if not __has_bzip2_support__ or os.getenv('NO_PLUGINS'): +if not __has_bzip2_support__ or os.getenv('NO_PLUGINS') or not nc.has_bzip2_filter(): test_files.remove('tst_compression_bzip2.py') sys.stdout.write('not running tst_compression_bzip2.py ...\n') -if not __has_blosc_support__ or os.getenv('NO_PLUGINS'): +if not __has_blosc_support__ or os.getenv('NO_PLUGINS') or not nc.has_blosc_filter(): test_files.remove('tst_compression_blosc.py') sys.stdout.write('not running tst_compression_blosc.py ...\n') -if not __has_szip_support__: +if not __has_szip_support__ or not nc.has_szip_filter(): test_files.remove('tst_compression_szip.py') sys.stdout.write('not running tst_compression_szip.py ...\n') +nc.close() +os.remove(filename) # Don't run tests that require network connectivity if os.getenv('NO_NET'): diff --git a/test/tst_compression_zstd.py b/test/tst_compression_zstd.py index 0e14436e6..1971dbfa6 100644 --- a/test/tst_compression_zstd.py +++ b/test/tst_compression_zstd.py @@ -51,7 +51,7 @@ def runTest(self): if __name__ == '__main__': nc = Dataset(filename1,'w') if not nc.has_zstd_filter(): - sys.stdout.write('blosc filter not available, skipping tests ...\n') + sys.stdout.write('zstd filter not available, skipping tests ...\n') else: nc.close() unittest.main() From 561c16bcc877cc3bd9759cd1ce06d9ad031b3c85 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Mon, 27 Jun 2022 18:44:31 -0600 Subject: [PATCH 11/13] test skipping plugin tests --- .github/workflows/build_master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index 00b3a52d4..4b1490bda 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -50,7 +50,7 @@ jobs: - name: Install netcdf4-python run: | export PATH=${NETCDF_DIR}/bin:${PATH} - export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir + #export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir python setup.py install - name: Test run: | From c7e46cbacfe11ac61abddc09cdd0ab630b26f997 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Mon, 27 Jun 2022 18:46:52 -0600 Subject: [PATCH 12/13] update --- Changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Changelog b/Changelog index 08d7e2cfd..2c673e30e 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,8 @@ + since version 1.6.0 release +============================== + * add Dataset methods has__filter (where =zstd,blosc,bzip2,szip) + to check for availability of extra compression filters. + version 1.6.0 (tag v1.6.0rel) ============================== * add support for new quantization functionality in netcdf-c 4.9.0 via "signficant_digits" From e117f851365f4465a6ab459d338ccb25c1deaddb Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Mon, 27 Jun 2022 18:53:19 -0600 Subject: [PATCH 13/13] revert --- .github/workflows/build_master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index 4b1490bda..00b3a52d4 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -50,7 +50,7 @@ jobs: - name: Install netcdf4-python run: | export PATH=${NETCDF_DIR}/bin:${PATH} - #export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir + export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir python setup.py install - name: Test run: |