Merge pull request #2625 from djhoese/bugfix-np2-compat

Cleanup various warnings encountered during tests

doc/source/conf.py

@@ -18,8 +18,6 @@
 import sys
 from datetime import datetime
 
-from pkg_resources import get_distribution
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -28,12 +26,14 @@
 
 from reader_table import generate_reader_table  # noqa: E402
 
+import satpy  # noqa: E402
+
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # get version using setuptools-scm
-release = get_distribution("satpy").version
+release = satpy.__version__
 # The full version, including alpha/beta/rc tags.
 # for example take major/minor
 version = ".".join(release.split(".")[:2])
@@ -44,7 +44,7 @@ class Mock(object):  # noqa
 
     def __init__(self, *args, **kwargs):
         """Mask any arguments to mock object."""
-        pass
+        self.__version__ = "0.0.0"
 
     def __call__(self, *args, **kwargs):
         """Mock a function and class object when accessed from mocked module."""

satpy/readers/abi_l1b.py

@@ -163,7 +163,7 @@ def _ir_calibrate(self, data):
 
         if self.clip_negative_radiances:
             min_rad = self._get_minimum_radiance(data)
-            data = data.clip(min=min_rad)
+            data = data.clip(min=data.dtype.type(min_rad))
 
         res = (fk2 / np.log(fk1 / data + 1) - bc1) / bc2
         res.attrs = data.attrs

satpy/readers/fci_l1c_nc.py

@@ -334,11 +334,10 @@ def _get_dataset_measurand(self, key, info=None):
         vr = attrs.get("valid_range", [-np.inf, np.inf])
         if key["calibration"] == "counts":
             attrs["_FillValue"] = fv
-            nfv = fv
+            nfv = data.dtype.type(fv)
         else:
             nfv = np.nan
-        data = data.where(data >= vr[0], nfv)
-        data = data.where(data <= vr[1], nfv)
+        data = data.where((data >= vr[0]) & (data <= vr[1]), nfv)
 
         res = self.calibrate(data, key)
 

satpy/readers/goes_imager_hrit.py

@@ -31,6 +31,7 @@
 import numpy as np
 import xarray as xr
 
+from satpy._compat import ArrayLike
 from satpy.readers._geos_area import get_area_definition, get_area_extent, get_geos_area_naming
 from satpy.readers.eum_base import recarray2dict, time_cds_short
 from satpy.readers.hrit_base import (
@@ -115,30 +116,40 @@ class CalibrationError(Exception):
                      ("msecs", "u1")])
 
 
-def make_sgs_time(sgs_time_array):
+def make_sgs_time(sgs_time_array: ArrayLike) -> datetime:
     """Make sgs time."""
-    year = ((sgs_time_array["century"] >> 4) * 1000 +
-            (sgs_time_array["century"] & 15) * 100 +
-            (sgs_time_array["year"] >> 4) * 10 +
-            (sgs_time_array["year"] & 15))
-    doy = ((sgs_time_array["doy1"] >> 4) * 100 +
-           (sgs_time_array["doy1"] & 15) * 10 +
-           (sgs_time_array["doy_hours"] >> 4))
-    hours = ((sgs_time_array["doy_hours"] & 15) * 10 +
-             (sgs_time_array["hours_mins"] >> 4))
-    mins = ((sgs_time_array["hours_mins"] & 15) * 10 +
-            (sgs_time_array["mins_secs"] >> 4))
-    secs = ((sgs_time_array["mins_secs"] & 15) * 10 +
-            (sgs_time_array["secs_msecs"] >> 4))
-    msecs = ((sgs_time_array["secs_msecs"] & 15) * 100 +
-             (sgs_time_array["msecs"] >> 4) * 10 +
-             (sgs_time_array["msecs"] & 15))
-    return (datetime(int(year), 1, 1) +
-            timedelta(days=int(doy - 1),
-                      hours=int(hours),
-                      minutes=int(mins),
-                      seconds=int(secs),
-                      milliseconds=int(msecs)))
+    epoch_year = _epoch_year_from_sgs_time(sgs_time_array)
+    doy_offset = _epoch_doy_offset_from_sgs_time(sgs_time_array)
+    return epoch_year + doy_offset
+
+
+def _epoch_year_from_sgs_time(sgs_time_array: ArrayLike) -> datetime:
+    century = sgs_time_array["century"].astype(np.int64)
+    year = sgs_time_array["year"].astype(np.int64)
+    year = ((century >> 4) * 1000 + (century & 15) * 100 + (year >> 4) * 10 + (year & 15))
+    return datetime(int(year), 1, 1)
+
+
+def _epoch_doy_offset_from_sgs_time(sgs_time_array: ArrayLike) -> timedelta:
+    doy1 = sgs_time_array["doy1"].astype(np.int64)
+    doy_hours = sgs_time_array["doy_hours"].astype(np.int64)
+    hours_mins = sgs_time_array["hours_mins"].astype(np.int64)
+    mins_secs = sgs_time_array["mins_secs"].astype(np.int64)
+    secs_msecs = sgs_time_array["secs_msecs"].astype(np.int64)
+    msecs = sgs_time_array["msecs"].astype(np.int64)
+
+    doy = ((doy1 >> 4) * 100 + (doy1 & 15) * 10 + (doy_hours >> 4))
+    hours = ((doy_hours & 15) * 10 + (hours_mins >> 4))
+    mins = ((hours_mins & 15) * 10 + (mins_secs >> 4))
+    secs = ((mins_secs & 15) * 10 + (secs_msecs >> 4))
+    msecs = ((secs_msecs & 15) * 100 + (msecs >> 4) * 10 + (msecs & 15))
+    return timedelta(
+        days=int(doy - 1),
+        hours=int(hours),
+        minutes=int(mins),
+        seconds=int(secs),
+        milliseconds=int(msecs)
+    )
 
 
 satellite_status = np.dtype([("TagType", "<u4"),

satpy/readers/hdfeos_base.py

@@ -285,7 +285,8 @@ def _get_good_data_mask(self, data_arr, is_category=False):
         if is_category and np.issubdtype(data_arr.dtype, np.integer):
             # no need to mask, the fill value is already what it needs to be
             return None, None
-        new_fill = np.nan
+        fill_type = data_arr.dtype.type if np.issubdtype(data_arr.dtype, np.floating) else np.float32
+        new_fill = fill_type(np.nan)
         data_arr.attrs.pop("_FillValue", None)
         good_mask = data_arr != fill_value
         return good_mask, new_fill

satpy/readers/mersi_l1b.py

@@ -141,7 +141,7 @@ def _mask_data(self, data, dataset_id, attrs):
         if dataset_id.get("calibration") == "counts":
             # preserve integer type of counts if possible
             attrs["_FillValue"] = fill_value
-            new_fill = fill_value
+            new_fill = data.dtype.type(fill_value)
         else:
             new_fill = np.nan
         if valid_range is not None:

satpy/readers/utils.py

@@ -459,7 +459,7 @@ def apply_earthsun_distance_correction(reflectance, utc_date=None):
     reflectance.attrs["sun_earth_distance_correction_applied"] = True
     reflectance.attrs["sun_earth_distance_correction_factor"] = sun_earth_dist
     with xr.set_options(keep_attrs=True):
-        reflectance = reflectance * sun_earth_dist * sun_earth_dist
+        reflectance = reflectance * reflectance.dtype.type(sun_earth_dist * sun_earth_dist)
     return reflectance
 
 
@@ -472,5 +472,5 @@ def remove_earthsun_distance_correction(reflectance, utc_date=None):
     reflectance.attrs["sun_earth_distance_correction_applied"] = False
     reflectance.attrs["sun_earth_distance_correction_factor"] = sun_earth_dist
     with xr.set_options(keep_attrs=True):
-        reflectance = reflectance / (sun_earth_dist * sun_earth_dist)
+        reflectance = reflectance / reflectance.dtype.type(sun_earth_dist * sun_earth_dist)
     return reflectance

satpy/readers/viirs_atms_sdr_base.py

@@ -100,11 +100,11 @@ def __init__(self, filename, filename_info, filetype_info, **kwargs):
         super().__init__(filename, filename_info, filetype_info, **kwargs)
 
     def _parse_datetime(self, datestr, timestr):
-        try:
-            datetime_str = datestr + timestr
-        except TypeError:
-            datetime_str = (str(datestr.data.compute().astype(str)) +
-                            str(timestr.data.compute().astype(str)))
+        if not isinstance(datestr, str):
+            datestr = str(datestr.data.compute().astype(str))
+        if not isinstance(timestr, str):
+            timestr = str(timestr.data.compute().astype(str))
+        datetime_str = datestr + timestr
 
         time_val = datetime.strptime(datetime_str, "%Y%m%d%H%M%S.%fZ")
         if abs(time_val - NO_DATE) < EPSILON_TIME:

satpy/tests/modifier_tests/test_parallax.py

@@ -30,6 +30,7 @@
 from pyresample import create_area_def
 
 import satpy.resample
+from satpy.tests.utils import xfail_skyfield_unstable_numpy2
 from satpy.writers import get_enhanced_image
 
 # NOTE:
@@ -438,6 +439,7 @@ def test_correct_area_cloudy_same_area(self, ):
         corrector = ParallaxCorrection(area)
         corrector(sc["CTH_constant"])
 
+    @pytest.mark.xfail(xfail_skyfield_unstable_numpy2(), reason="Skyfield doesn't support numpy 2 yet")
     def test_correct_area_no_orbital_parameters(self, caplog, fake_tle):
         """Test ParallaxCorrection when CTH has no orbital parameters.
 
@@ -761,6 +763,7 @@ def fake_scene(self, yaml_code):
                 "area": area})
         return sc
 
+    @pytest.mark.xfail(xfail_skyfield_unstable_numpy2(), reason="Skyfield doesn't support numpy 2 yet")
     def test_double_load(self, fake_scene, conf_file, fake_tle):
         """Test that loading corrected and uncorrected works correctly.
 
@@ -790,6 +793,7 @@ def test_no_compute(self, fake_scene, conf_file):
             sccc.return_value = [os.fspath(conf_file)]
             fake_scene.load(["parallax_corrected_VIS006"])
 
+    @pytest.mark.xfail(xfail_skyfield_unstable_numpy2(), reason="Skyfield doesn't support numpy 2 yet")
     def test_enhanced_image(self, fake_scene, conf_file, fake_tle):
         """Test that image enhancement is the same."""
         with unittest.mock.patch(

satpy/tests/reader_tests/_li_test_utils.py

@@ -370,13 +370,13 @@ def l2_lfl_schema(settings=None):
                 "default_data": lambda: np.random.uniform(stime, etime, nobs)
             },
             "l1b_geolocation_warning": {
-                "format": "u8",
+                "format": "i1",
                 "shape": ("flashes",),
                 "long_name": "L1b geolocation warning",
                 "default_data": lambda: -127
             },
             "l1b_radiometric_warning": {
-                "format": "u8",
+                "format": "i1",
                 "shape": ("flashes",),
                 "long_name": "L1b radiometric warning",
                 "default_data": lambda: -127

satpy/tests/reader_tests/modis_tests/test_modis_l1b.py

@@ -77,6 +77,10 @@ def _load_and_check_geolocation(scene, resolution, exp_res, exp_shape, has_res,
         assert lat_arr.shape == exp_shape
         # compute lon/lat at the same time to avoid wasted computation
         lon_vals, lat_vals = dask.compute(lon_arr, lat_arr)
+        assert lon_arr.dtype == lat_arr.dtype
+        assert lon_arr.dtype == np.float32
+        assert lon_vals.dtype == lon_arr.dtype
+        assert lat_vals.dtype == lat_arr.dtype
         np.testing.assert_array_less(lon_vals, 0)
         np.testing.assert_array_less(0, lat_vals)
         check_callback(lon_arr)

satpy/tests/reader_tests/test_ami_l1b.py

@@ -65,7 +65,7 @@ def setUp(self, xr_, counts=None):
         if counts is None:
             rad_data = (np.arange(10.).reshape((2, 5)) + 1.) * 50.
             rad_data = (rad_data + 1.) / 0.5
-            rad_data = rad_data.astype(np.int16)
+            rad_data = rad_data.astype(np.uint16)
             counts = xr.DataArray(
                 da.from_array(rad_data, chunks="auto"),
                 dims=("y", "x"),

satpy/tests/reader_tests/test_gerb_l2_hr_h5.py

@@ -22,6 +22,7 @@
 import pytest
 
 from satpy import Scene
+from satpy.tests.utils import xfail_h5py_unstable_numpy2
 
 FNAME = "G4_SEV4_L20_HR_SOL_TH_20190606_130000_V000.hdf"
 
@@ -120,6 +121,7 @@ def gerb_l2_hr_h5_dummy_file(tmp_path_factory):
     return filename
 
 
+@pytest.mark.xfail(xfail_h5py_unstable_numpy2(), reason="h5py doesn't include numpy 2 fix")
 @pytest.mark.parametrize("name", ["Solar Flux", "Thermal Flux", "Solar Radiance", "Thermal Radiance"])
 def test_dataset_load(gerb_l2_hr_h5_dummy_file, name):
     """Test loading the solar flux component."""

satpy/tests/test_utils.py

@@ -29,6 +29,7 @@
 import pytest
 import xarray as xr
 
+from satpy.tests.utils import xfail_skyfield_unstable_numpy2
 from satpy.utils import (
     angle2xyz,
     get_legacy_chunk_size,
@@ -202,6 +203,7 @@ def test_get_satpos_fails_with_informative_error(self, attrs):
         with pytest.raises(KeyError, match="Unable to determine satellite position.*"):
             get_satpos(data_arr)
 
+    @pytest.mark.xfail(xfail_skyfield_unstable_numpy2(), reason="Skyfield does not support numpy 2 yet")
     def test_get_satpos_from_satname(self, caplog):
         """Test getting satellite position from satellite name only."""
         import pyorbital.tlefile

satpy/tests/utils.py

@@ -407,3 +407,34 @@ def assert_attrs_equal(attrs, attrs_exp, tolerance=0):
                 )
             except TypeError:
                 assert attrs[key] == attrs_exp[key], err_msg
+
+
+def xfail_skyfield_unstable_numpy2():
+    """Determine if skyfield-based tests should be xfail in the unstable numpy 2.x environment."""
+    try:
+        import skyfield
+
+        # known numpy incompatibility:
+        from skyfield import timelib  # noqa
+    except ImportError:
+        skyfield = None
+
+    import os
+    is_unstable_ci = os.environ.get("UNSTABLE", "0") in ("1", "true")
+    is_np2 = np.__version__.startswith("2.")
+    return skyfield is None and is_np2 and is_unstable_ci
+
+
+def xfail_h5py_unstable_numpy2():
+    """Determine if h5py-based tests should be xfail in the unstable numpy 2.x environment."""
+    from packaging import version
+    try:
+        import h5py
+        is_broken_h5py = version.parse(h5py.__version__) <= version.parse("3.10.0")
+    except ImportError:
+        is_broken_h5py = True
+
+    import os
+    is_unstable_ci = os.environ.get("UNSTABLE", "0") in ("1", "true")
+    is_np2 = np.__version__.startswith("2.")
+    return is_broken_h5py and is_np2 and is_unstable_ci

satpy/tests/writer_tests/test_awips_tiled.py

@@ -495,9 +495,10 @@ def test_multivar_numbered_tiles_glm(self, sector, extra_kwargs, tmp_path):
             "_FillValue": 1,
         })
 
-        w.save_datasets([ds1, ds2, ds3, dqf], sector_id="TEST", source_name="TESTS",
-                        tile_count=(3, 3), template="glm_l2_rad{}".format(sector.lower()),
-                        **extra_kwargs)
+        with pytest.warns(UserWarning, match="Production location attribute "):
+            w.save_datasets([ds1, ds2, ds3, dqf], sector_id="TEST", source_name="TESTS",
+                            tile_count=(3, 3), template="glm_l2_rad{}".format(sector.lower()),
+                            **extra_kwargs)
         fn_glob = self._get_glm_glob_filename(extra_kwargs)
         all_files = glob(os.path.join(str(tmp_path), fn_glob))
         assert len(all_files) == 9

satpy/writers/awips_tiled.py

@@ -620,9 +620,12 @@ def _get_factor_offset_fill(input_data_arr, vmin, vmax, encoding):
         # file data type to allow for extra fill values
         num_fills = 0
 
-    if is_unsigned or unsigned_in_signed:
+    if is_unsigned:
         # max value
         fills = [2 ** file_bit_depth - 1]
+    elif unsigned_in_signed:
+        # max unsigned value is -1 as a signed int
+        fills = [-1]
     else:
         # max value
         fills = [2 ** (file_bit_depth - 1) - 1]