From 792644ab9f0edf3e2e4c6d27a4a5bf2ba3e8cf57 Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Thu, 1 Feb 2024 11:39:56 +0100 Subject: [PATCH] fix up reader and tests --- pyaerocom/plugins/icos/reader.py | 22 +++++++++++++++------- tests/plugins/icos/test_reader.py | 15 ++++++++------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/pyaerocom/plugins/icos/reader.py b/pyaerocom/plugins/icos/reader.py index ef0357a76..a581beb87 100644 --- a/pyaerocom/plugins/icos/reader.py +++ b/pyaerocom/plugins/icos/reader.py @@ -101,18 +101,18 @@ def read( first_file: int | None = None, last_file: int | None = None, ) -> UngriddedData: - if len(var_to_retrieve) > 1: - raise NotImplementedError("Unnskyld, ReadICOS can only read one variable at a time...") - if var_to_retrieve is None: var_to_retrieve = self.DEFAULT_VARS[0] - if unsupported := set(var_to_retrieve) - set(self.PROVIDES_VARIABLES): - raise ValueError(f"Unsupported variables: {', '.join(sorted(unsupported))}") - if isinstance(var_to_retrieve, list): var_to_retrieve = var_to_retrieve[0] + if len([var_to_retrieve]) > 1: + raise NotImplementedError("Unnskyld, ReadICOS can only read one variable at a time...") + + if unsupported := set([var_to_retrieve]) - set(self.PROVIDES_VARIABLES): + raise ValueError(f"Unsupported variables: {', '.join(sorted(unsupported))}") + if files is not None and not isinstance(files, tuple): files = tuple(files) @@ -146,9 +146,17 @@ def read( return UngriddedData.from_station_data(stations) else: stations: list[StationData] = [] + this_var_files = sorted( + Path(self.data_dir).rglob(self.FILEMASK_MAPPING[var_to_retrieve]) + ) + for station_name, paths in self.stations(files).items(): + paths_to_read = list(set(paths) & set(this_var_files)) + if not paths_to_read: + continue + logger.debug(f"Reading station {station_name}") - ds = self._read_dataset(paths)[var_to_retrieve] + ds = self._read_dataset(paths) ds = ds.rename({self.VAR_MAPPING[var_to_retrieve]: var_to_retrieve}) ds = ds.assign( time=self._dataset_time(ds), diff --git a/tests/plugins/icos/test_reader.py b/tests/plugins/icos/test_reader.py index 438eed239..11e4453bc 100644 --- a/tests/plugins/icos/test_reader.py +++ b/tests/plugins/icos/test_reader.py @@ -16,7 +16,7 @@ VARS_DEFAULT = {"vmrco2", "vmrch4", "vmrco"} VARS_PROVIDED = VARS_DEFAULT # | {} add more if ever needed -station_names = pytest.mark.parametrize("station", ("bir", "gat", "hpb")) +station_names = pytest.mark.parametrize("station", ("Birkenes", "Gartow", "Hohenpeissenberg")) @pytest.fixture(scope="module") @@ -63,13 +63,13 @@ def test_stations(reader: ReadICOS, station: str): def test_PROVIDES_VARIABLES(reader: ReadICOS): - return set(reader.PROVIDES_VARIABLES) >= VARS_PROVIDED + assert set(reader.PROVIDES_VARIABLES) >= VARS_PROVIDED @station_names def test_read_file(reader: ReadICOS, station_files: list[str]): data = reader.read_file(station_files[-1]) - assert set(data.contains_vars) == VARS_DEFAULT + assert set(data.contains_vars) <= VARS_DEFAULT def test_read_file_error(reader: ReadICOS): @@ -81,16 +81,17 @@ def test_read_file_error(reader: ReadICOS): @station_names def test_read(reader: ReadICOS, station_files: list[str]): - data = reader.read(VARS_PROVIDED, station_files, first_file=0, last_file=1) - assert set(data.contains_vars) == VARS_PROVIDED + for var in VARS_PROVIDED: + data = reader.read(var, station_files, first_file=0, last_file=1) + assert set(data.contains_vars) <= VARS_PROVIDED def test_read_error(reader: ReadICOS): bad_variable_name = "not-a-variable" with pytest.raises(ValueError) as e: - reader.read((bad_variable_name,)) + reader.read(bad_variable_name) assert str(e.value) == f"Unsupported variables: {bad_variable_name}" -def test_reader_gives_correct_mep_path(reader: ReadICOS, icos_path: Path): +def test_reader_gives_correct_icos_path(reader: ReadICOS, icos_path: Path): assert reader.data_dir == str(icos_path)