Default worker sections

configs/clevr/default_clevr.yml

@@ -33,6 +33,7 @@ test:
     #resize_image: [224, 224]
 
 pipeline:
+  name: tmp
   disable: image_viewer
 
   label_to_target:

configs/default/workers/offline_trainer.yml

@@ -1,9 +1,9 @@
 ####################################################################
 # Section defining all the default values of parameters used during training when using ptp-offline-trainer.
 
-# If you want to use different section for training pass its name as command line argument '--training_section_name' to trainer (DEFAULT: training)
-# Note: in such a case remember to define all the required parameters in the new section.
-training:
+# If you want to use different section for "training" pass its name as command line argument '--training_section_name' to trainer (DEFAULT: training)
+# Note: the following parameters will be (anyway) used as default values.
+default_training:
   # Set the random seeds: -1 means that they will be picked randomly.
   # Note: their final values will be stored in the final training_configuration.yml saved to log dir.
   seed_numpy: -1
@@ -69,8 +69,8 @@ training:
 ####################################################################
 # Section defining all the default values of parameters used during validation.
 # If you want to use different section for validation pass its name as command line argument '--validation_section_name' to trainer (DEFAULT: validation)
-# Note: in such a case remember to define all the required parameters in the new section.
-validation:
+# Note: the following parameters will be (anyway) used as default values.
+default_validation:
   # Defines how often the partial validation will be performed.
   # In this trainer Partial Validation is optional (negative value means it is disabled)
   partial_validation_interval: -1

configs/default/workers/online_trainer.yml

@@ -1,9 +1,8 @@
 ####################################################################
 # Section defining all the default values of parameters used during training when using ptp-online-trainer.
-
-# If you want to use different section for training pass its name as command line argument '--training_section_name' to trainer (DEFAULT: training)
-# Note: in such a case remember to define all the required parameters in the new section.
-training:
+# If you want to use different section for "training" pass its name as command line argument '--training_section_name' to trainer (DEFAULT: training)
+# Note: the following parameters will be (anyway) used as default values.
+default_training:
   # Set the random seeds: -1 means that they will be picked randomly.
   # Note: their final values will be stored in the final training_configuration.yml saved to log dir.
   seed_numpy: -1
@@ -69,8 +68,8 @@ training:
 ####################################################################
 # Section defining all the default values of parameters used during validation.
 # If you want to use different section for validation pass its name as command line argument '--validation_section_name' to trainer (DEFAULT: validation)
-# Note: in such a case remember to define all the required parameters in the new section.
-validation:
+# Note: the following parameters will be (anyway) used as default values.
+default_validation:
   # Defines how often the partial validation will be performed.
   # In this trainer Partial Validation is mandatory, hence interval must be > 0.
   partial_validation_interval: 100

configs/default/workers/processor.yml

@@ -1,10 +1,9 @@
 ####################################################################
 # Section defining all the default values of parameters used during testing.
-# If you want to use different section for training pass its name as command line argument '--section_name' to trainer (DEFAULT: test)
-# Note: in such a case remember to define all the required parameters in the new section.
-test:
+# If you want to use different section during "processing" pass its name as command line argument '--section_name' to trainer (DEFAULT: test)
+# Note: the following parameters will be (anyway) used as default values.
+default_test:
   # Set the random seeds: -1 means that they will be picked randomly.
-  # Note: their final values will be stored in the final training_configuration.yml saved to log dir.
   seed_numpy: -1
   seed_torch: -1
 

ptp/configuration/config_interface.py

@@ -265,7 +265,11 @@ def del_default_params(self, key):
         :type key: str
 
         """
-        self._config_registry.del_default_params(self._keys_path + [key])
+        if type(key) is list:
+            keypath = self._keys_path + key
+        else:
+            keypath = self._keys_path + [key]
+        self._config_registry.del_default_params(keypath)
 
     def del_config_params(self, key):
         """
@@ -277,7 +281,11 @@ def del_config_params(self, key):
         :type key: str
 
         """
-        self._config_registry.del_config_params(self._keys_path + [key])
+        if type(key) is list:
+            keypath = self._keys_path + key
+        else:
+            keypath = self._keys_path + [key]
+        self._config_registry.del_config_params(keypath)
 
     def add_config_params_from_yaml(self, yaml_path: str):
         """
@@ -296,50 +304,3 @@ def add_config_params_from_yaml(self, yaml_path: str):
 
         # add config param
         self.add_config_params(params_from_yaml)
-
-
-if __name__ == '__main__':
-    # Test code
-    pi0 = ConfigInterface()
-    pi1 = ConfigInterface('level0', 'level1')
-
-    pi0.add_default_params({
-        'param0': "0_from_code",
-        'param1': "1_from_code"
-    })
-
-    print('pi0', pi0.to_dict())
-
-    pi0.add_config_params({
-        'param1': "-1_from_config_file"
-    })
-
-    print('pi0', pi0.to_dict())
-
-    pi1.add_default_params({
-        'param2': 2,
-        'param3': 3
-    })
-
-    print('pi0', pi0.to_dict())
-    print('pi1', pi1.to_dict())
-
-    pi1.add_config_params({
-        'param2': -2
-    })
-
-    print('pi0', pi0.to_dict())
-    print('pi1', pi1.to_dict())
-
-    pi2 = pi0['level0']
-    print('pi2', pi2.to_dict())
-
-    pi1.add_config_params({
-        'param2': -3
-    })
-
-    print('pi2', pi2.to_dict())
-
-    pi3 = pi0['level0']['level1']
-
-    print('pi3', pi3.to_dict())

ptp/configuration/config_registry.py

@@ -246,7 +246,7 @@ def lookup_recursion(dic, key, *keys):
             return dic[key]
 
         lookup_keys = keypath[:-1]  # We keep the last key for use with `del`
-        if len(keypath) > 0:
+        if len(lookup_keys) > 0:
             r = lookup_recursion(current_dict, *lookup_keys)
             del r[keypath[-1]]
         else:

ptp/workers/processor.py

@@ -72,6 +72,10 @@ def setup_global_experiment(self):
         """
         # Call base method to parse all command line arguments and add default sections.
         super(Processor, self).setup_experiment()
+
+        # "Pass" configuration parameters from the default_test section to section indicated by the section_name.
+        self.config.add_default_params({ self.app_state.args.section_name:  self.config['default_test'].to_dict()} )
+        self.config.del_default_params('default_test')
 
         # Retrieve checkpoint file.
         chkpt_file = self.app_state.args.load_checkpoint

ptp/workers/trainer.py

@@ -124,6 +124,15 @@ def setup_experiment(self):
         # Call base method to parse all command line arguments and add default sections.
         super(Trainer, self).setup_experiment()
 
+        # "Pass" configuration parameters from the "default_training" section to training section indicated by the section_name.
+        self.config.add_default_params({ self.app_state.args.training_section_name :  self.config['default_training'].to_dict()} )
+        self.config.del_default_params('default_training')
+
+        # "Pass" configuration parameters from the "default_validation" section to validation section indicated by the section_name.
+        self.config.add_default_params({ self.app_state.args.validation_section_name:  self.config['default_validation'].to_dict()} )
+        self.config.del_default_params('default_validation')
+
+
         # Check the presence of the CUDA-compatible devices.
         if self.app_state.args.use_gpu and (torch.cuda.device_count() == 0):
             self.logger.error("Cannot use GPU as there are no CUDA-compatible devices present in the system!")

tests/__init__.py

@@ -1,5 +1,6 @@
 from .app_state_tests import TestAppState
 from .component_tests import TestComponent
+from .config_interface_tests import TestConfigInterface
 from .config_registry_tests import TestConfigRegistry
 from .data_dict_tests import TestDataDict
 from .data_definition_tests import TestDataDefinition
@@ -13,6 +14,7 @@
     'TestAppState',
     'TestComponent',
     'TestConfigRegistry',
+    'TestConfigInterface',
     'TestDataDict',
     'TestDataDefinition',
     'TestHandshaking',

tests/config_interface_tests.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) tkornuta, IBM Corporation 2019
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Tomasz Kornuta"
+
+import unittest
+
+from ptp.configuration.config_interface import ConfigInterface
+
+class TestConfigInterface(unittest.TestCase):
+
+    def test_default_params(self):
+        config = ConfigInterface()
+        # Add params - first method.
+        config.add_default_params({'default_0': {'default_1': 'str'}})
+        self.assertNotEqual(config['default_0'], None)
+        self.assertEqual(config['default_0']['default_1'], 'str')
+
+        # Remove params - first method.
+        config.del_default_params(['default_0', 'default_1'])
+        with self.assertRaises(KeyError):
+            _ = config['default_0']['default_1']
+
+        # Add params - second method.
+        config['default_0'].add_default_params({'default_2': 'str'})
+
+        # Remove params - second method.
+        config['default_0'].del_default_params('default_2')
+        with self.assertRaises(KeyError):
+            _ = config['default_0']['default_2']
+
+        # Add 3rd parameter under 0.
+        config['default_0'].add_default_params({'default_3': 'str'})
+
+        # Remove the main section.
+        config.del_default_params('default_0')
+        with self.assertRaises(KeyError):
+            _ = config['default_0']
+
+
+    def test_config_params(self):
+        config = ConfigInterface()
+        # Add params.
+        config.add_config_params({'config_0': {'config_1': 'int'}})
+        self.assertNotEqual(config['config_0'], None)
+        self.assertEqual(config['config_0']['config_1'], 'int')
+
+        # Remove params.
+        config.del_config_params(['config_0', 'config_1'])
+        with self.assertRaises(KeyError):
+            _ = config['config_0']['config_1']
+
+    def test_overwrite_params(self):
+        config = ConfigInterface()
+        config.add_config_params({'under': True})
+        config.add_default_params({'under': False})
+        self.assertEqual(config['under'], True)
+
+#if __name__ == "__main__":
+#    unittest.main()

tests/config_registry_tests.py

@@ -34,6 +34,7 @@ def test_default_params(self):
         with self.assertRaises(KeyError):
             _ = config['default_0']['default_1']
 
+
     def test_config_params(self):
         config = ConfigRegistry()
         # Add params.
@@ -46,11 +47,13 @@ def test_config_params(self):
         with self.assertRaises(KeyError):
             _ = config['config_0']['config_1']
 
+
     def test_overwrite_params(self):
         config = ConfigRegistry()
         config.add_config_params({'under': True})
         config.add_default_params({'under': False})
         self.assertEqual(config['under'], True)
 
+
 #if __name__ == "__main__":
 #    unittest.main()