Added dotdict format feature for JSON file.

JsonPreprocessor/CJsonPreprocessor.py

@@ -182,8 +182,8 @@ def __init__(self, syntax: CSyntaxType = CSyntaxType.json , currentCfg : dict =
         self.currentCfg = currentCfg
         self.lUpdatedParams = []
         self.lNestedParams = []
+        self.lDotInParamName = []
 
-
     def __sNormalizePath(self, sPath : str) -> str:
         """
    Python struggles with
@@ -340,7 +340,6 @@ def __nestedParamHandler(self, sInputStr : str) -> str:
       String which contains the resolved variable.  
         '''
 
-        #globals().update(currentCfg)
         referVars = re.findall('(\${\s*.*?\s*})', sInputStr)
         if len(referVars) > 1:
             sUpdateVar =  referVars[0]
@@ -384,7 +383,17 @@ def __nestedParamHandler(self, sInputStr : str) -> str:
             pattern = '(\\' + referVars[0] + '\s*\[\s*.*?\s*\])'
             variable = re.findall(pattern, sInputStr)
             if variable == []:
-                sStrHandled = referVars[0]
+                if "." in referVars[0]:
+                    dotdictVariable = re.sub('\${\s*(.*?)\s*}', '\\1', referVars[0])
+                    lDotdictVariable = dotdictVariable.split(".")
+                    lParams = self.__handleDotdictFormat(lDotdictVariable, [])
+                    sStrHandled = '${' + lParams[0] + '}'
+                    lParams.pop(0)
+                    for item in lParams:
+                        sStrHandled = sStrHandled + "['" + item + "']"
+                else:
+                    sStrHandled = referVars[0]
+
                 return sStrHandled
             else:
                 fullVariable = variable[0]
@@ -396,7 +405,46 @@ def __nestedParamHandler(self, sInputStr : str) -> str:
                 sStrHandled = fullVariable
                 return sStrHandled
 
-    def __updateAndReplaceNestedParam(self, oJson : dict, recursive : bool = False):
+    def __handleDotdictFormat(self, lInputListParams : list, lParams: list = []) -> list:
+        '''
+   This method checks the availability of param names contained "." in dotdict format element in JSON config file.
+
+**Args:**
+
+   **lInputListParams** (*list*)
+
+      List of items which separated by "." of dotdict format element.
+
+   **lParams** (*list*)
+
+      List of param names in dotdict format element.
+
+**Returns:**
+
+   **lParams** (*list*)
+
+        '''
+        checkParam = lInputListParams[0]
+        i = 0
+        bDotdictParam = False
+        for item in lInputListParams:
+            if i > 0:
+                checkParam = checkParam + '.' + item
+                if checkParam in self.lDotInParamName:
+                    lParams.append(checkParam)
+                    bDotdictParam = True
+                    lInputListParams = lInputListParams[i+1:]
+                    break
+            i+=1
+        if not bDotdictParam:
+            lParams.append(lInputListParams[0])
+            lInputListParams.pop(0)
+        if lInputListParams == []:
+            return lParams
+        else:
+            return self.__handleDotdictFormat(lInputListParams, lParams)    
+
+    def __updateAndReplaceNestedParam(self, oJson : dict, bNested : bool = False, recursive : bool = False):
         '''
    This method replaces all nested parameters in key and value of a json object .
 
@@ -429,9 +477,8 @@ def __jsonUpdated(k, v, oJson):
             for k, v in self.currentCfg.items():
                 globals().update({k:v})
 
-        tmpJson = copy.deepcopy(oJson)            
+        tmpJson = copy.deepcopy(oJson)
         for k, v in tmpJson.items():
-            bNested = False
             if re.match('.*\${\s*', k.lower()):
                 if re.match("str\(\s*\${.+", k.lower()):
                     k = re.sub("str\(\s*(\${.+)\s*\)", "\\1", k)
@@ -441,15 +488,15 @@ def __jsonUpdated(k, v, oJson):
                 bNested = True
 
             if isinstance(v, dict):
-                v = self.__updateAndReplaceNestedParam(v, recursive=True)
-                __jsonUpdated(k, v, tmpJson)
-                bNested = False
+                v, bNested = self.__updateAndReplaceNestedParam(v, bNested, recursive=True)
+                __jsonUpdated(k, v, oJson) if bNested else __jsonUpdated(k, v, tmpJson)
 
             elif isinstance(v, list):
                 tmpValue = []
                 for item in v:
                     if isinstance(item, str) and re.match('^.*\s*\${\s*', item.lower()):
                         bStringValue = False
+                        bNested = True
                         if re.match("str\(\s*\${.+", item.lower()):
                             item = re.sub("str\(\s*(\${.+)\s*\)", "\\1", item)
                             bStringValue = True
@@ -472,8 +519,7 @@ def __jsonUpdated(k, v, oJson):
                             raise Exception(f"The variable '{tmpItemAfterProcessed}' is not available!")
 
                     tmpValue.append(item)
-                __jsonUpdated(k, v, oJson)
-                bNested = False
+                __jsonUpdated(k, tmpValue, oJson)
 
             elif isinstance(v, str):
                 if re.match('^.*\s*\${\s*', v.lower()):
@@ -503,15 +549,15 @@ def __jsonUpdated(k, v, oJson):
                         v = '\"' + v + '\"'
 
                     __jsonUpdated(k, v, oJson)
-                    bNested = False
+                    bNested = True
                 else:
                     __jsonUpdated(k, v, oJson)
                     bNested = False
             else:
                 if bNested:
                     __jsonUpdated(k, v, oJson)
                     bNested = False
-        return oJson
+        return oJson, bNested
 
     def __checkAndUpdateKeyValue(self, sInputStr: str) -> str:
         '''
@@ -543,6 +589,23 @@ def __checkAndUpdateKeyValue(self, sInputStr: str) -> str:
         sOutput = sInputStr
         return sOutput
 
+    def __checkDotInParamName(self, oJson : dict):
+        '''
+   This is recrusive funtion collects all parameters which contain "." in the name.
+
+**Args:**
+   **oJson** (*dict*)
+   Json object which want to collect all parameter's name contained "."
+
+**Returns:**
+   **None**
+        '''
+        for k, v in oJson.items():
+            if "." in k and k not in self.lDotInParamName:
+                self.lDotInParamName.append(k)
+            if isinstance(v, dict):
+                self.__checkDotInParamName(v)
+
     def jsonLoad(self, jFile : str, masterFile : bool = True):
         '''
    This function is the entry point of JsonPreprocessor.
@@ -646,20 +709,11 @@ def __handleStrNoneTrueFalse(objJson):
         oJson = __handleStrNoneTrueFalse(oJson)        
         os.chdir(currentDir)
 
+        self.__checkDotInParamName(oJson)
+
         if masterFile:
             for k, v in oJson.items():
                 globals().update({k:v})
 
-            # Checking availability of nested parameters before updating and replacing.
-            for param in self.lNestedParams:
-                parseNestedParam = self.__nestedParamHandler(param)
-                tmpParseNestedParam = re.sub('\\${\s*(.*?)\s*}', '\\1', parseNestedParam)
-                sExec = "value = " + tmpParseNestedParam if isinstance(tmpParseNestedParam, str) else \
-                        "value = " + str(tmpParseNestedParam)
-                try:
-                    ldict = {}
-                    exec(sExec, globals(), ldict)
-                except:
-                    raise Exception(f"The variable '{tmpParseNestedParam}' is not available!")
-            oJson = self.__updateAndReplaceNestedParam(oJson)
+            oJson, bNested = self.__updateAndReplaceNestedParam(oJson)
         return oJson

atest/jsonpreprocessor/test_jsonpreprocessor.py

@@ -483,3 +483,50 @@ def test_utf8_encoding_imported_06(self):
         assert oJsonData['utf8']['日本'] == 1.987
         assert oJsonData['utf8']['हिंदी'] == "นี่คือการทดสอบตัวเอง UTF-8"
         assert oJsonData['utf8']['한국인'] == "1"
+
+class TestDotdictFormat:
+
+    def test_dotdict_format_in_value_01(self):
+        '''
+        Test nested param with dotdict format in the right site of a colon in a JSON file.
+        '''
+        sJsonfile = os.path.abspath("../testdata/config/09_dotdict_format/dotdict_format_config_01.json")
+        oJsonPreprocessor = CJsonPreprocessor(syntax="python")
+        oJsonData = oJsonPreprocessor.jsonLoad(sJsonfile)
+        assert oJsonData['params']['global']['newParam_01'] == 1
+        assert oJsonData['params']['global']['newParamStr_01'] == "1"
+        assert oJsonData['params']['newParam_02'] == "Structure test"
+        assert oJsonData['newParam_03'] == "general"
+        assert oJsonData['newParam_04'] == "This is a string"
+
+    def test_dotdict_format_in_value_02(self):
+        '''
+        Test nested param with dotdict format in the right site of a colon in a JSON file.
+        Nested param is a element of a list
+        '''
+        sJsonfile = os.path.abspath("../testdata/config/09_dotdict_format/dotdict_format_config_02.json")
+        oJsonPreprocessor = CJsonPreprocessor(syntax="python")
+        oJsonData = oJsonPreprocessor.jsonLoad(sJsonfile)
+        assert oJsonData['params']['global']['newListParam_01'] == ['one', 2, 1, 'three']
+        assert oJsonData['params']['global']['newListParam_02'] == ['one', 2, 'This is a string', 'three', 'Structure test']
+
+    def test_dotdict_format_in_value_03(self):
+        '''
+        Test nested param with dotdict format in the left site of a colon in a JSON file.
+        '''
+        sJsonfile = os.path.abspath("../testdata/config/09_dotdict_format/dotdict_format_config_03.json")
+        oJsonPreprocessor = CJsonPreprocessor(syntax="python")
+        oJsonData = oJsonPreprocessor.jsonLoad(sJsonfile)
+        assert oJsonData['params']['global']['gPrepro.Float.Param'] == 9.999
+        assert oJsonData['params']['global']['gPrepro.Structure']['test.Structure'] == "Change value"
+        assert oJsonData['params']['global']['gPrepro.Structure']['general'] == "Update value"
+
+    def test_dotdict_format_in_value_04(self):
+        '''
+        Test nested param with dotdict format in the both sites of a colon in a JSON file.
+        '''
+        sJsonfile = os.path.abspath("../testdata/config/09_dotdict_format/dotdict_format_config_04.json")
+        oJsonPreprocessor = CJsonPreprocessor(syntax="python")
+        oJsonData = oJsonPreprocessor.jsonLoad(sJsonfile)
+        assert oJsonData['params']['global']['gPrepro.Float.Param'] == 1
+        assert oJsonData['params']['global']['gPrepro.Structure']['test.Structure'] == "Structure test"

atest/testdata/config/09_dotdict_format/dotdict_format_config_01.json

@@ -0,0 +1,28 @@
+//  Copyright 2020-2022 Robert Bosch Car Multimedia GmbH
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+{
+    "WelcomeString": "Hello... JsonPreprocessor selftest is running now!",
+    "params": {
+      "global": {
+        "[import]" : "./import/dotdict_format_common.json",
+        "newParam_01" : ${params.global.gPreprol.IntParam},
+        "newParamStr_01" : "${params.global.gPreprol.IntParam}"
+      },
+      "newParam_02" : ${params.global.gPrepro.Structure.general}
+    },
+    "newParam_03" : ${params.global.gPrepro.Structure.test.Structure},
+    "newParam_04" : ${params.global.g.Prepro.String.test},
+    "abc" : "This is a multline string\nwith\nhttp://www.google.de\na link inside",
+    "Target//Name" : "gen3flex//dlt"
+  }

atest/testdata/config/09_dotdict_format/dotdict_format_config_02.json

@@ -0,0 +1,25 @@
+//  Copyright 2020-2022 Robert Bosch Car Multimedia GmbH
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+{
+    "WelcomeString": "Hello... JsonPreprocessor selftest is running now!",
+    "params": {
+      "global": {
+        "[import]" : "./import/dotdict_format_common.json",
+        "newListParam_01" : ["one", 2, ${params.global.gPreprol.IntParam}, "three"],
+        "newListParam_02" : ["one", 2, ${params.global.g.Prepro.String.test}, "three", ${params.global.gPrepro.Structure.general}]
+      }
+    },
+    "abc" : "This is a multline string\nwith\nhttp://www.google.de\na link inside",
+    "Target//Name" : "gen3flex//dlt"
+  }

atest/testdata/config/09_dotdict_format/dotdict_format_config_03.json

@@ -0,0 +1,26 @@
+//  Copyright 2020-2022 Robert Bosch Car Multimedia GmbH
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+{
+    "WelcomeString": "Hello... JsonPreprocessor selftest is running now!",
+    "params": {
+      "global": {
+        "[import]" : "./import/dotdict_format_common.json"
+      }
+    },
+    ${params.global.gPrepro.Float.Param} : 9.999,
+    ${params.global.gPrepro.Structure.test.Structure} : "Change value",
+    ${params.global.gPrepro.Structure.general} : "Update value",
+    "abc" : "This is a multline string\nwith\nhttp://www.google.de\na link inside",
+    "Target//Name" : "gen3flex//dlt"
+  }

atest/testdata/config/09_dotdict_format/dotdict_format_config_04.json

@@ -0,0 +1,25 @@
+//  Copyright 2020-2022 Robert Bosch Car Multimedia GmbH
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+{
+    "WelcomeString": "Hello... JsonPreprocessor selftest is running now!",
+    "params": {
+      "global": {
+        "[import]" : "./import/dotdict_format_common.json"
+      }
+    },
+    ${params.global.gPrepro.Float.Param} : ${params.global.gPreprol.IntParam},
+    ${params.global.gPrepro.Structure.test.Structure} : ${params.global.gPrepro.Structure.general},
+    "abc" : "This is a multline string\nwith\nhttp://www.google.de\na link inside",
+    "Target//Name" : "gen3flex//dlt"
+  }

atest/testdata/config/09_dotdict_format/import/dotdict_format_common.json

@@ -0,0 +1,28 @@
+//  Copyright 2020-2022 Robert Bosch Car Multimedia GmbH
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//*****************************************************************************
+
+{
+    "gPreprol.IntParam" : 1,
+
+    "gPrepro.Float.Param" : 1.332,
+
+    "g.Prepro.String.test"   : "This is a string",
+
+    "gPrepro.Structure": {
+                           "test.Structure": "general",
+                           "general": "Structure test"
+                        },
+    "scope.Check": "For testing purpose"
+  }