Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix ABI readers using wrong dtype for resolution-based chunks #2627

Merged
merged 1 commit into from
Nov 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions satpy/readers/abi_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,10 @@ def _chunk_bytes_for_resolution(self) -> int:
# this is true for all CSPP Geo GRB output (226 for all sectors) and full disk from other sources
# 250 has been seen for AWS/CLASS CONUS, Mesoscale 1, and Mesoscale 2 files
# we align this with 4 on-disk chunks at 500m, so it will be 2 on-disk chunks for 1km, and 1 for 2km
high_res_elems_disk_aligned = np.round(max(num_high_res_elems_per_dim / (4 * 226), 1)) * (4 * 226)
high_res_elems_disk_aligned = round(max(num_high_res_elems_per_dim / (4 * 226), 1)) * (4 * 226)
low_res_factor = int(self.filetype_info.get("resolution", 2000) // 500)
res_elems_per_dim = int(high_res_elems_disk_aligned / low_res_factor)
return (res_elems_per_dim ** 2) * 4
return (res_elems_per_dim ** 2) * 2 # 16-bit integers on disk

@staticmethod
def _rename_dims(nc):
Expand Down
26 changes: 15 additions & 11 deletions satpy/tests/reader_tests/test_abi_l1b.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,18 +136,14 @@ def generate_l1b_filename(chan_name: str) -> str:

@pytest.fixture()
def c01_refl(tmp_path) -> xr.DataArray:
# 4 bytes for 32-bit floats
# 4 on-disk chunks for 500 meter data
# 226 on-disk chunk size
# Square (**2) for 2D size
with dask.config.set({"array.chunk-size": ((226 * 4) ** 2) * 4}):
with _apply_dask_chunk_size():
reader = _create_reader_for_data(tmp_path, "C01", None, 1000)
return reader.load(["C01"])["C01"]


@pytest.fixture()
def c01_rad(tmp_path) -> xr.DataArray:
with dask.config.set({"array.chunk-size": ((226 * 4) ** 2) * 4}):
with _apply_dask_chunk_size():
reader = _create_reader_for_data(tmp_path, "C01", None, 1000)
return reader.load([DataQuery(name="C01", calibration="radiance")])["C01"]

Expand All @@ -169,14 +165,14 @@ def c01_rad_h5netcdf(tmp_path) -> xr.DataArray:
"valid_range": (0, 4095),
},
)
with dask.config.set({"array.chunk-size": ((226 * 4) ** 2) * 4}):
with _apply_dask_chunk_size():
reader = _create_reader_for_data(tmp_path, "C01", rad, 1000)
return reader.load([DataQuery(name="C01", calibration="radiance")])["C01"]


@pytest.fixture()
def c01_counts(tmp_path) -> xr.DataArray:
with dask.config.set({"array.chunk-size": ((226 * 4) ** 2) * 4}):
with _apply_dask_chunk_size():
reader = _create_reader_for_data(tmp_path, "C01", None, 1000)
return reader.load([DataQuery(name="C01", calibration="counts")])["C01"]

Expand All @@ -187,7 +183,7 @@ def _load_data_array(
clip_negative_radiances: bool = False,
):
rad = _fake_c07_data()
with dask.config.set({"array.chunk-size": ((226 * 4) ** 2) * 4}):
with _apply_dask_chunk_size():
reader = _create_reader_for_data(
tmp_path,
"C07",
Expand Down Expand Up @@ -241,14 +237,22 @@ def _create_reader_for_data(
return load_readers([str(data_path)], "abi_l1b", reader_kwargs=reader_kwargs)["abi_l1b"]


def _apply_dask_chunk_size():
# 226 on-disk chunk size
# 8 on-disk chunks for 500 meter data
# Square (**2) for 2D size
# 4 bytes for 32-bit floats
return dask.config.set({"array.chunk-size": ((226 * 8) ** 2) * 4})


def _get_and_check_array(data_arr: xr.DataArray, exp_dtype: npt.DTypeLike) -> npt.NDArray:
data_np = data_arr.data.compute()
assert isinstance(data_arr, xr.DataArray)
assert isinstance(data_arr.data, da.Array)
assert isinstance(data_np, np.ndarray)
res = 1000 if RAD_SHAPE[1000][0] == data_np.shape[0] else 2000
assert data_arr.chunks[0][0] == 226 * (4 / (res / 500))
assert data_arr.chunks[1][0] == 226 * (4 / (res / 500))
assert data_arr.chunks[0][0] == 226 * (8 / (res / 500))
assert data_arr.chunks[1][0] == 226 * (8 / (res / 500))

assert data_np.dtype == data_arr.dtype
assert data_np.dtype == exp_dtype
Expand Down
Loading