Merge pull request #156 from pysat/bug/numpy_pysatcdf

BUG/TST: update test behavior in the absence/presence of pysatCDF

.github/workflows/main.yml

@@ -15,7 +15,7 @@ jobs:
         numpy_ver: ["latest"]
         include:
           - python-version: "3.8"
-            numpy_ver: "1.20"
+            numpy_ver: "1.21"
             os: "ubuntu-latest"
 
     name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }}
@@ -35,11 +35,11 @@ jobs:
       if: ${{ matrix.numpy_ver != 'latest'}}
       run: |
         pip install --no-binary :numpy: numpy==${{ matrix.numpy_ver }}
+        pip install pysatCDF --no-binary=pysatCDF
 
     - name: Install standard dependencies
       run: |
         pip install -r requirements.txt
-        pip install pysatCDF --no-binary=pysatCDF
         pip install -r test_requirements.txt
 
     - name: Set up pysat

CHANGELOG.md

@@ -26,9 +26,9 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * Added unit tests for the different platform method attributes
   * xarray support for TIMED SEE
 * Maintenance
-  * Added a version cap for numpy (required for cdf interface, revisit before
-    release)
-  * Updated actions and templates based on pysatEcosystem docs.
+  * Removed duplicate tests if pysatCDF not isntalled
+  * Only test pysatCDF on GitHub Actions for older numpy versions
+  * Updated actions and templates based on pysatEcosystem docs
   * Remove pandas cap on NEP29 tests
   * Updated dosctring style for consistency
   * Removed version cap for xarray

README.md

@@ -82,7 +82,8 @@ For data products stored as CDF files, this package can use either `cdflib` or
 `pysatCDF`.  Note that `cdflib` is a pure python package and more readily
 deployable across systems, whereas `pysatCDF` interfaces with the fortran.  
 This is a faster approach for loading data, but may not install on all systems.  
-Therefore, `pysatCDF` is optional rather than required.
+There are known issues with `numpy`>=1.24. Therefore, `pysatCDF` is optional
+rather than required.  
 
 You can specify which load routine to use via the optional `use_cdflib` kwarg.
 If no kwarg is specified, `pysatNASA` will default to `pysatCDF` if it is

pysatNASA/instruments/methods/cdaweb.py

@@ -124,8 +124,10 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1),
 
     Note
     ----
-    This routine is intended to be used by pysat instrument modules supporting
-    a particular NASA CDAWeb dataset
+    - This routine is intended to be used by pysat instrument modules supporting
+      a particular NASA CDAWeb dataset
+    - pysatCDF (as of v0.3.2) does not support numpy>=1.24.  Load errors may
+      arise.  See https://github.com/pysat/pysatCDF/issues/46
 
     """
 

pysatNASA/tests/test_instruments.py

@@ -16,6 +16,15 @@
 # Import the test classes from pysat
 from pysat.tests.classes import cls_instrument_library as clslib
 
+try:
+    import pysatCDF  # noqa: F401
+    # If this successfully imports, tests need to be run with both pysatCDF
+    # and cdflib
+    cdflib_only = False
+except ImportError:
+    # pysatCDF is not present, standard tests default to cdflib.
+    cdflib_only = True
+
 
 # Tell the standard tests which instruments to run each test on.
 # Need to return instrument list for custom tests.
@@ -27,7 +36,9 @@
 for inst in instruments['download']:
     fname = inst['inst_module'].supported_tags[inst['inst_id']][inst['tag']]
     if '.cdf' in fname:
-        instruments['cdf'].append(inst)
+        temp_inst, _ = clslib.initialize_test_inst_and_date(inst)
+        if temp_inst.pandas_format:
+            instruments['cdf'].append(inst)
 
 
 class TestInstruments(clslib.InstLibTests):
@@ -42,6 +53,9 @@ class TestInstruments(clslib.InstLibTests):
 
     @pytest.mark.second
     @pytest.mark.parametrize("inst_dict", instruments['cdf'])
+    @pytest.mark.skipif(cdflib_only,
+                        reason=" ".join(("Additional load tests not required",
+                                         "when pysatCDF not installed")))
     def test_load_cdflib(self, inst_dict):
         """Test that instruments load at each cleaning level.
 

requirements.txt

@@ -3,7 +3,7 @@ requests
 beautifulsoup4
 lxml
 cdflib>=0.4.4
-numpy<1.24
+numpy
 pandas
 pysat>=3.0.4
 cdasws