Merge pull request #979 from metno/icosco2

Fix up ICOS reader and tests to work with CO2, CH3, and CO

pyaerocom/plugins/icos/reader.py

@@ -101,18 +101,18 @@ def read(
         first_file: int | None = None,
         last_file: int | None = None,
     ) -> UngriddedData:
-        if len(var_to_retrieve) > 1:
-            raise NotImplementedError("Unnskyld, ReadICOS can only read one variable at a time...")
-
         if var_to_retrieve is None:
             var_to_retrieve = self.DEFAULT_VARS[0]
 
-        if unsupported := set(var_to_retrieve) - set(self.PROVIDES_VARIABLES):
-            raise ValueError(f"Unsupported variables: {', '.join(sorted(unsupported))}")
-
         if isinstance(var_to_retrieve, list):
             var_to_retrieve = var_to_retrieve[0]
 
+        if len([var_to_retrieve]) > 1:
+            raise NotImplementedError("Unnskyld, ReadICOS can only read one variable at a time...")
+
+        if unsupported := set([var_to_retrieve]) - set(self.PROVIDES_VARIABLES):
+            raise ValueError(f"Unsupported variables: {', '.join(sorted(unsupported))}")
+
         if files is not None and not isinstance(files, tuple):
             files = tuple(files)
 
@@ -146,9 +146,17 @@ def read(
             return UngriddedData.from_station_data(stations)
         else:
             stations: list[StationData] = []
+            this_var_files = sorted(
+                Path(self.data_dir).rglob(self.FILEMASK_MAPPING[var_to_retrieve])
+            )
+
             for station_name, paths in self.stations(files).items():
+                paths_to_read = list(set(paths) & set(this_var_files))
+                if not paths_to_read:
+                    continue
+
                 logger.debug(f"Reading station {station_name}")
-                ds = self._read_dataset(paths)[var_to_retrieve]
+                ds = self._read_dataset(paths)
                 ds = ds.rename({self.VAR_MAPPING[var_to_retrieve]: var_to_retrieve})
                 ds = ds.assign(
                     time=self._dataset_time(ds),

tests/plugins/icos/test_reader.py

@@ -16,7 +16,7 @@
 VARS_DEFAULT = {"vmrco2", "vmrch4", "vmrco"}
 VARS_PROVIDED = VARS_DEFAULT  # | {} add more if ever needed
 
-station_names = pytest.mark.parametrize("station", ("bir", "gat", "hpb"))
+station_names = pytest.mark.parametrize("station", ("Birkenes", "Gartow", "Hohenpeissenberg"))
 
 
 @pytest.fixture(scope="module")
@@ -69,7 +69,7 @@ def test_PROVIDES_VARIABLES(reader: ReadICOS):
 @station_names
 def test_read_file(reader: ReadICOS, station_files: list[str]):
     data = reader.read_file(station_files[-1])
-    assert set(data.contains_vars) == VARS_DEFAULT
+    assert set(data.contains_vars) <= VARS_DEFAULT
 
 
 def test_read_file_error(reader: ReadICOS):
@@ -81,16 +81,17 @@ def test_read_file_error(reader: ReadICOS):
 
 @station_names
 def test_read(reader: ReadICOS, station_files: list[str]):
-    data = reader.read(VARS_PROVIDED, station_files, first_file=0, last_file=1)
-    assert set(data.contains_vars) == VARS_PROVIDED
+    for var in VARS_PROVIDED:
+        data = reader.read(var, station_files, first_file=0, last_file=1)
+        assert set(data.contains_vars) <= VARS_PROVIDED
 
 
 def test_read_error(reader: ReadICOS):
     bad_variable_name = "not-a-variable"
     with pytest.raises(ValueError) as e:
-        reader.read((bad_variable_name,))
+        reader.read(bad_variable_name)
     assert str(e.value) == f"Unsupported variables: {bad_variable_name}"
 
 
-def test_reader_gives_correct_mep_path(reader: ReadICOS, icos_path: Path):
+def test_reader_gives_correct_icos_path(reader: ReadICOS, icos_path: Path):
     assert reader.data_dir == str(icos_path)