[pysiral] bugfix in config files and tests to identify them

CITATION.cff

@@ -13,5 +13,5 @@ authors:
     given-names: Stephan
 title: "pysiral: Product update (C3S, AWI, CCI, CryoTEMPO)"
 version: v0.9.6
-doi: 10.5281/zenodo.5566347 
+doi: 10.5281/zenodo.5566347
 date-released: 2017-12-18

pysiral/__init__.py

@@ -25,7 +25,7 @@
 from attrdict import AttrDict
 from distutils import dir_util
 from loguru import logger
-from typing import Union
+from typing import Union, Iterable
 
 import warnings
 warnings.filterwarnings("ignore")
@@ -496,6 +496,28 @@ def get_processor_definition_ids(self, processor_level):
         ids = self.get_yaml_setting_filelist(lookup_directory, return_value="ids")
         return ids
 
+    def get_settings_files(self, settings_type: str, data_level: str) -> Iterable[Path]:
+        """
+        Returns all processor settings or output definitions files for a given data level.
+        :param settings_type:
+        :param data_level:
+        :return:
+        """
+
+        if settings_type not in self.VALID_SETTING_TYPES:
+            return []
+
+        if data_level not in self.VALID_DATA_LEVEL_IDS:
+            return []
+
+        # Get all settings files in settings/{data_level} and its
+        # subdirectories
+        lookup_directory = self.get_local_setting_path(settings_type, data_level)
+        _, files = self.get_yaml_setting_filelist(lookup_directory)
+
+        # Test if ids are unique and return error for the moment
+        return files
+
     def get_settings_file(self, settings_type, data_level, setting_id_or_filename):
         """ Returns a processor settings file for a given data level.
         (data level: l2 or l3). The second argument can either be an

pysiral/resources/pysiral-cfg/output/l2p/l2p_awi_v2p4.yaml

@@ -95,7 +95,7 @@ variables:
     valid_max: 2
     flag_values: "0, 1, 2"
     flag_meanings: "pulse_limited_lrm doppler_delay_sar doppler_delay_sar_interferometric"
-    units: "1"
+    units: 1
     coordinates: time
 
   region_code:
@@ -104,8 +104,8 @@ variables:
     valid_min: 0
     valid_max: 21
     flag_values: "0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 21"
-    flag_meaning:"inland_sea_and_lakes ice_free_oceans sea_of_okhotsk bering hudson_bay st_john baffin_bay east_greenland_sea barents_sea kara_sea laptev_sea east_siberian_sea chukchi_sea beaufort_sea canadian_islands central_arctic baltic_sea land land_outline"
-    units: "1"
+    flag_meaning: "inland_sea_and_lakes ice_free_oceans sea_of_okhotsk bering hudson_bay st_john baffin_bay east_greenland_sea barents_sea kara_sea laptev_sea east_siberian_sea chukchi_sea beaufort_sea canadian_islands central_arctic baltic_sea land land_outline"
+    units: 1
     coordinates: time
 
   radar_freeboard:
@@ -135,15 +135,15 @@ variables:
   sea_ice_type:
     long_name: "fraction of multi-year ice (0: only first year ice, 1: only multi-year ice)"
     standard_name: sea_ice_classification
-    units: "1"
+    units: 1
     valid_min: 0.0
     valid_max: 1.0
     coordinates: time
 
   sea_ice_type_uncertainty:
     long_name: "uncertainty of sea ice classification"
     standard_name: sea_ice_classification standard_error
-    units: "1"
+    units: 1
     coordinates: time
 
   snow_depth:

pysiral/resources/pysiral-cfg/output/l3/l3_default.yaml

@@ -24,7 +24,7 @@ global_attributes:
   tracking_id: ""
   conventions: "CF-1.6"
   product_version: "1.0"
-  summary	string: ""
+  summary: ""
   keywords: "Sea ice thickness, Arctic Ocean (Antarctic Ocean), radar altimetry"
   id: ""
   naming: ""
@@ -35,8 +35,8 @@ global_attributes:
   date_issued: ""
   date_modified: ""
   creator_name: "Alfred Wegener Institute, Helmholtz Centre for Polar and Marine Research"
-  creator_url	string: "http://www.awi.de"
-  creator_email	string: "TBD"
+  creator_url: "http://www.awi.de"
+  creator_email: "TBD"
   project: ""
   geospatial_lat_min: "{geospatial_lat_min}"
   geospatial_lat_max: "{geospatial_lat_max}"
@@ -47,7 +47,7 @@ global_attributes:
   time_coverage_duration: "13 years"
   time_coverage_resolution: Monthly" 
   standard_name_vocabulary: ""
-  license	string: "n.a."
+  license: "n.a."
   platform: "{mission_name}"
   sensor: "{mission_sensor}"
   snow_source: "{source_snow}"

tests/test_definition_files.py

@@ -49,19 +49,8 @@ def testAuxdataDefinitionContent(self):
         since they might be different)
         :return:
         """
-        current_config_target = psrlcfg.current_config_target
-        for config_target in psrlcfg.VALID_CONFIG_TARGETS:
-            if config_target == "USER_HOME" and not Path(psrlcfg.userhome_config_path).is_dir():
-                continue
-            psrlcfg.set_config_target(config_target)
-            psrlcfg.reload()
-            self._testAuxdataDefinitionContent(config_target)
-        psrlcfg.set_config_target(current_config_target)
-        psrlcfg.reload()
-
-    def _testAuxdataDefinitionContent(self, target):
         for category, id, item in psrlcfg.auxdef.items:
-            aux_id = "{}:{}:{}".format(target, category, id)
+            aux_id = "{}:{}".format(category, id)
             config_dict_keys = item.keys
             for required_key in ["options", "long_name", "pyclass", "local_repository"]:
                 self.assertTrue(required_key in config_dict_keys, msg="{} has {}".format(aux_id, required_key))

tests/test_l1_proc_def_files.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""
+Testing all Level-2 output definition files for compabilility with
+Level2Processor conventions
+
+#TODO: Add CF/ACDD compliance checks
+
+@author: Stefan Hendricks
+"""
+
+import unittest
+from attrdict import AttrDict
+
+from pysiral import psrlcfg
+from pysiral.config import get_yaml_config
+from loguru import logger
+logger.disable("pysiral")
+
+
+class TestL1ProcDef(unittest.TestCase):
+
+    def setUp(self):
+
+        # Get a list of processor definition files in the code repository
+        self.l1_proc_files = psrlcfg.get_settings_files("output", "l1")
+
+    def testYamlSyntaxOfDefinitionFiles(self):
+        for filename in self.l1_proc_files:
+            content = get_yaml_config(filename)
+            self.assertIsInstance(content, AttrDict, msg=filename)
+
+
+if __name__ == '__main__':
+    suite = unittest.TestLoader().loadTestsFromTestCase(TestL1ProcDef)
+    unittest.TextTestRunner(verbosity=2).run(suite)

tests/test_l2_ouput_def_files.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+"""
+Testing all Level-2 output definition files for compabilility with
+Level2Processor conventions
+
+#TODO: Add CF/ACDD compliance checks
+
+@author: Stefan Hendricks
+"""
+
+import unittest
+from attrdict import AttrDict
+
+from pysiral import psrlcfg
+from pysiral.config import get_yaml_config
+from loguru import logger
+logger.disable("pysiral")
+
+
+class TestL2OutputDef(unittest.TestCase):
+
+    def setUp(self):
+
+        # Get a list of processor definition files in the code repository
+        l2i_files = psrlcfg.get_settings_files("output", "l2i")
+        l2p_files = psrlcfg.get_settings_files("output", "l2p")
+        self.l2_output_files = l2i_files + l2p_files
+
+    def testYamlSyntaxOfDefinitionFiles(self):
+        for filename in self.l2_output_files:
+            content = get_yaml_config(filename)
+            self.assertIsInstance(content, AttrDict, msg=filename)
+
+
+if __name__ == '__main__':
+    suite = unittest.TestLoader().loadTestsFromTestCase(TestL2OutputDef)
+    unittest.TextTestRunner(verbosity=2).run(suite)

tests/test_l2proc_def_files.py → tests/test_l2_proc_def_files.py

@@ -11,6 +11,7 @@
 from attrdict import AttrDict
 
 from pysiral import psrlcfg
+from pysiral.config import get_yaml_config
 from pysiral.l2proc.procsteps import Level2ProcessorStepOrder
 from loguru import logger
 logger.disable("pysiral")
@@ -24,6 +25,11 @@ def setUp(self):
         l2proc_ids = psrlcfg.get_setting_ids("proc", "l2")
         self.l2procdef_files = [psrlcfg.get_settings_file("proc", "l2", l2proc_id) for l2proc_id in l2proc_ids]
 
+    def testYamlSyntaxOfDefinitionFiles(self):
+        for filename in self.l2procdef_files:
+            content = get_yaml_config(filename)
+            self.assertIsInstance(content, AttrDict, msg=filename)
+
     def testConfigFileRootTags(self):
         required_tags = ["metadata", "auxdata", "procsteps"]
         for l2procdef_file in self.l2procdef_files:

tests/test_l3_ouput_def_files.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""
+Testing all Level-3 output definition files for compabilility with
+Level2Processor conventions
+
+#TODO: Add CF/ACDD compliance checks
+
+@author: Stefan Hendricks
+"""
+
+import unittest
+from attrdict import AttrDict
+
+from pysiral import psrlcfg
+from pysiral.config import get_yaml_config
+from loguru import logger
+logger.disable("pysiral")
+
+
+class TestL3OutputDef(unittest.TestCase):
+
+    def setUp(self):
+
+        # Get a list of processor definition files in the code repository
+        self.l3_output_files = psrlcfg.get_settings_files("output", "l3")
+
+    def testYamlSyntaxOfDefinitionFiles(self):
+        for filename in self.l3_output_files:
+            content = get_yaml_config(filename)
+            self.assertIsInstance(content, AttrDict, msg=filename)
+
+
+if __name__ == '__main__':
+    suite = unittest.TestLoader().loadTestsFromTestCase(TestL3OutputDef)
+    unittest.TextTestRunner(verbosity=2).run(suite)

tests/test_l3_proc_def_files.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""
+Testing all Level-2 processor definition files for compabilility with
+Level2Processor conventions
+
+@author: Stefan Hendricks
+"""
+
+import yaml
+import unittest
+from attrdict import AttrDict
+
+from pysiral import psrlcfg
+from pysiral.config import get_yaml_config
+from pysiral.l2proc.procsteps import Level2ProcessorStepOrder
+from loguru import logger
+logger.disable("pysiral")
+
+
+class TestL3ProcDef(unittest.TestCase):
+
+    def setUp(self):
+
+        # Get a list of processor definition files in the code repository
+        self.l2procdef_files = psrlcfg.get_settings_files("proc", "l3")
+
+    def testYamlSyntaxOfDefinitionFiles(self):
+        for filename in self.l2procdef_files:
+            content = get_yaml_config(filename)
+            self.assertIsInstance(content, AttrDict, msg=filename)
+
+
+if __name__ == '__main__':
+    suite = unittest.TestLoader().loadTestsFromTestCase(TestL3ProcDef)
+    unittest.TextTestRunner(verbosity=2).run(suite)