From 0aa2b58bb86990c01a3a03d6021dbf508adbeea4 Mon Sep 17 00:00:00 2001 From: mas2hc Date: Tue, 4 Apr 2023 21:02:24 +0700 Subject: [PATCH 1/3] Added dotdict format feature for JSON file. --- JsonPreprocessor/CJsonPreprocessor.py | 95 +++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 14 deletions(-) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index bc1ff4f8..acdd53f5 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -182,8 +182,8 @@ def __init__(self, syntax: CSyntaxType = CSyntaxType.json , currentCfg : dict = self.currentCfg = currentCfg self.lUpdatedParams = [] self.lNestedParams = [] + self.lDotInParamName = [] - def __sNormalizePath(self, sPath : str) -> str: """ Python struggles with @@ -340,7 +340,6 @@ def __nestedParamHandler(self, sInputStr : str) -> str: String which contains the resolved variable. ''' - #globals().update(currentCfg) referVars = re.findall('(\${\s*.*?\s*})', sInputStr) if len(referVars) > 1: sUpdateVar = referVars[0] @@ -384,7 +383,17 @@ def __nestedParamHandler(self, sInputStr : str) -> str: pattern = '(\\' + referVars[0] + '\s*\[\s*.*?\s*\])' variable = re.findall(pattern, sInputStr) if variable == []: - sStrHandled = referVars[0] + if "." in referVars[0]: + dotdictVariable = re.sub('\${\s*(.*?)\s*}', '\\1', referVars[0]) + lDotdictVariable = dotdictVariable.split(".") + lParams = self.__handleDotdictFormat(lDotdictVariable, []) + sStrHandled = '${' + lParams[0] + '}' + lParams.pop(0) + for item in lParams: + sStrHandled = sStrHandled + "['" + item + "']" + else: + sStrHandled = referVars[0] + return sStrHandled else: fullVariable = variable[0] @@ -396,7 +405,46 @@ def __nestedParamHandler(self, sInputStr : str) -> str: sStrHandled = fullVariable return sStrHandled - def __updateAndReplaceNestedParam(self, oJson : dict, recursive : bool = False): + def __handleDotdictFormat(self, lInputListParams : list, lParams: list = []) -> list: + ''' + This method checks the availability of param names contained "." in dotdict format element in JSON config file. + +**Args:** + + **lInputListParams** (*list*) + + List of items which separated by "." of dotdict format element. + + **lParams** (*list*) + + List of param names in dotdict format element. + +**Returns:** + + **lParams** (*list*) + + ''' + checkParam = lInputListParams[0] + i = 0 + bDotdictParam = False + for item in lInputListParams: + if i > 0: + checkParam = checkParam + '.' + item + if checkParam in self.lDotInParamName: + lParams.append(checkParam) + bDotdictParam = True + lInputListParams = lInputListParams[i+1:] + break + i+=1 + if not bDotdictParam: + lParams.append(lInputListParams[0]) + lInputListParams.pop(0) + if lInputListParams == []: + return lParams + else: + return self.__handleDotdictFormat(lInputListParams, lParams) + + def __updateAndReplaceNestedParam(self, oJson : dict, bNested : bool = False, recursive : bool = False): ''' This method replaces all nested parameters in key and value of a json object . @@ -429,9 +477,8 @@ def __jsonUpdated(k, v, oJson): for k, v in self.currentCfg.items(): globals().update({k:v}) - tmpJson = copy.deepcopy(oJson) + tmpJson = copy.deepcopy(oJson) for k, v in tmpJson.items(): - bNested = False if re.match('.*\${\s*', k.lower()): if re.match("str\(\s*\${.+", k.lower()): k = re.sub("str\(\s*(\${.+)\s*\)", "\\1", k) @@ -441,15 +488,15 @@ def __jsonUpdated(k, v, oJson): bNested = True if isinstance(v, dict): - v = self.__updateAndReplaceNestedParam(v, recursive=True) - __jsonUpdated(k, v, tmpJson) - bNested = False + v, bNested = self.__updateAndReplaceNestedParam(v, bNested, recursive=True) + __jsonUpdated(k, v, oJson) if bNested else __jsonUpdated(k, v, tmpJson) elif isinstance(v, list): tmpValue = [] for item in v: if isinstance(item, str) and re.match('^.*\s*\${\s*', item.lower()): bStringValue = False + bNested = True if re.match("str\(\s*\${.+", item.lower()): item = re.sub("str\(\s*(\${.+)\s*\)", "\\1", item) bStringValue = True @@ -472,8 +519,7 @@ def __jsonUpdated(k, v, oJson): raise Exception(f"The variable '{tmpItemAfterProcessed}' is not available!") tmpValue.append(item) - __jsonUpdated(k, v, oJson) - bNested = False + __jsonUpdated(k, tmpValue, oJson) elif isinstance(v, str): if re.match('^.*\s*\${\s*', v.lower()): @@ -503,7 +549,7 @@ def __jsonUpdated(k, v, oJson): v = '\"' + v + '\"' __jsonUpdated(k, v, oJson) - bNested = False + bNested = True else: __jsonUpdated(k, v, oJson) bNested = False @@ -511,7 +557,7 @@ def __jsonUpdated(k, v, oJson): if bNested: __jsonUpdated(k, v, oJson) bNested = False - return oJson + return oJson, bNested def __checkAndUpdateKeyValue(self, sInputStr: str) -> str: ''' @@ -543,6 +589,23 @@ def __checkAndUpdateKeyValue(self, sInputStr: str) -> str: sOutput = sInputStr return sOutput + def __checkDotInParamName(self, oJson : dict): + ''' + This is recrusive funtion collects all parameters which contain "." in the name. + +**Args:** + **oJson** (*dict*) + Json object which want to collect all parameter's name contained "." + +**Returns:** + **None** + ''' + for k, v in oJson.items(): + if "." in k and k not in self.lDotInParamName: + self.lDotInParamName.append(k) + if isinstance(v, dict): + self.__checkDotInParamName(v) + def jsonLoad(self, jFile : str, masterFile : bool = True): ''' This function is the entry point of JsonPreprocessor. @@ -646,6 +709,8 @@ def __handleStrNoneTrueFalse(objJson): oJson = __handleStrNoneTrueFalse(oJson) os.chdir(currentDir) + self.__checkDotInParamName(oJson) + if masterFile: for k, v in oJson.items(): globals().update({k:v}) @@ -661,5 +726,7 @@ def __handleStrNoneTrueFalse(objJson): exec(sExec, globals(), ldict) except: raise Exception(f"The variable '{tmpParseNestedParam}' is not available!") - oJson = self.__updateAndReplaceNestedParam(oJson) + oJson, bNested = self.__updateAndReplaceNestedParam(oJson) + # for k, v in oJson.items(): + # globals().update({k:v}) return oJson \ No newline at end of file From 9b5f16118cdbdb4b1bf21eec158b0f076fe39320 Mon Sep 17 00:00:00 2001 From: mas2hc Date: Wed, 5 Apr 2023 14:35:06 +0700 Subject: [PATCH 2/3] Added selftest for dotdict format feature. --- .../jsonpreprocessor/test_jsonpreprocessor.py | 47 +++++++++++++++++++ .../dotdict_format_config_01.json | 28 +++++++++++ .../dotdict_format_config_02.json | 25 ++++++++++ .../dotdict_format_config_03.json | 26 ++++++++++ .../dotdict_format_config_04.json | 25 ++++++++++ .../import/dotdict_format_common.json | 28 +++++++++++ 6 files changed, 179 insertions(+) create mode 100644 atest/testdata/config/09_dotdict_format/dotdict_format_config_01.json create mode 100644 atest/testdata/config/09_dotdict_format/dotdict_format_config_02.json create mode 100644 atest/testdata/config/09_dotdict_format/dotdict_format_config_03.json create mode 100644 atest/testdata/config/09_dotdict_format/dotdict_format_config_04.json create mode 100644 atest/testdata/config/09_dotdict_format/import/dotdict_format_common.json diff --git a/atest/jsonpreprocessor/test_jsonpreprocessor.py b/atest/jsonpreprocessor/test_jsonpreprocessor.py index 6b6cd25f..d8dbadf1 100644 --- a/atest/jsonpreprocessor/test_jsonpreprocessor.py +++ b/atest/jsonpreprocessor/test_jsonpreprocessor.py @@ -483,3 +483,50 @@ def test_utf8_encoding_imported_06(self): assert oJsonData['utf8']['日本'] == 1.987 assert oJsonData['utf8']['हिंदी'] == "นี่คือการทดสอบตัวเอง UTF-8" assert oJsonData['utf8']['한국인'] == "1" + +class TestDotdictFormat: + + def test_dotdict_format_in_value_01(self): + ''' + Test nested param with dotdict format in the right site of a colon in a JSON file. + ''' + sJsonfile = os.path.abspath("../testdata/config/09_dotdict_format/dotdict_format_config_01.json") + oJsonPreprocessor = CJsonPreprocessor(syntax="python") + oJsonData = oJsonPreprocessor.jsonLoad(sJsonfile) + assert oJsonData['params']['global']['newParam_01'] == 1 + assert oJsonData['params']['global']['newParamStr_01'] == "1" + assert oJsonData['params']['newParam_02'] == "Structure test" + assert oJsonData['newParam_03'] == "general" + assert oJsonData['newParam_04'] == "This is a string" + + def test_dotdict_format_in_value_02(self): + ''' + Test nested param with dotdict format in the right site of a colon in a JSON file. + Nested param is a element of a list + ''' + sJsonfile = os.path.abspath("../testdata/config/09_dotdict_format/dotdict_format_config_02.json") + oJsonPreprocessor = CJsonPreprocessor(syntax="python") + oJsonData = oJsonPreprocessor.jsonLoad(sJsonfile) + assert oJsonData['params']['global']['newListParam_01'] == ['one', 2, 1, 'three'] + assert oJsonData['params']['global']['newListParam_02'] == ['one', 2, 'This is a string', 'three', 'Structure test'] + + def test_dotdict_format_in_value_03(self): + ''' + Test nested param with dotdict format in the left site of a colon in a JSON file. + ''' + sJsonfile = os.path.abspath("../testdata/config/09_dotdict_format/dotdict_format_config_03.json") + oJsonPreprocessor = CJsonPreprocessor(syntax="python") + oJsonData = oJsonPreprocessor.jsonLoad(sJsonfile) + assert oJsonData['params']['global']['gPrepro.Float.Param'] == 9.999 + assert oJsonData['params']['global']['gPrepro.Structure']['test.Structure'] == "Change value" + assert oJsonData['params']['global']['gPrepro.Structure']['general'] == "Update value" + + def test_dotdict_format_in_value_04(self): + ''' + Test nested param with dotdict format in the both sites of a colon in a JSON file. + ''' + sJsonfile = os.path.abspath("../testdata/config/09_dotdict_format/dotdict_format_config_04.json") + oJsonPreprocessor = CJsonPreprocessor(syntax="python") + oJsonData = oJsonPreprocessor.jsonLoad(sJsonfile) + assert oJsonData['params']['global']['gPrepro.Float.Param'] == 1 + assert oJsonData['params']['global']['gPrepro.Structure']['test.Structure'] == "Structure test" \ No newline at end of file diff --git a/atest/testdata/config/09_dotdict_format/dotdict_format_config_01.json b/atest/testdata/config/09_dotdict_format/dotdict_format_config_01.json new file mode 100644 index 00000000..600927d6 --- /dev/null +++ b/atest/testdata/config/09_dotdict_format/dotdict_format_config_01.json @@ -0,0 +1,28 @@ +// Copyright 2020-2022 Robert Bosch Car Multimedia GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +{ + "WelcomeString": "Hello... JsonPreprocessor selftest is running now!", + "params": { + "global": { + "[import]" : "./import/dotdict_format_common.json", + "newParam_01" : ${params.global.gPreprol.IntParam}, + "newParamStr_01" : "${params.global.gPreprol.IntParam}" + }, + "newParam_02" : ${params.global.gPrepro.Structure.general} + }, + "newParam_03" : ${params.global.gPrepro.Structure.test.Structure}, + "newParam_04" : ${params.global.g.Prepro.String.test}, + "abc" : "This is a multline string\nwith\nhttp://www.google.de\na link inside", + "Target//Name" : "gen3flex//dlt" + } \ No newline at end of file diff --git a/atest/testdata/config/09_dotdict_format/dotdict_format_config_02.json b/atest/testdata/config/09_dotdict_format/dotdict_format_config_02.json new file mode 100644 index 00000000..575f8338 --- /dev/null +++ b/atest/testdata/config/09_dotdict_format/dotdict_format_config_02.json @@ -0,0 +1,25 @@ +// Copyright 2020-2022 Robert Bosch Car Multimedia GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +{ + "WelcomeString": "Hello... JsonPreprocessor selftest is running now!", + "params": { + "global": { + "[import]" : "./import/dotdict_format_common.json", + "newListParam_01" : ["one", 2, ${params.global.gPreprol.IntParam}, "three"], + "newListParam_02" : ["one", 2, ${params.global.g.Prepro.String.test}, "three", ${params.global.gPrepro.Structure.general}] + } + }, + "abc" : "This is a multline string\nwith\nhttp://www.google.de\na link inside", + "Target//Name" : "gen3flex//dlt" + } \ No newline at end of file diff --git a/atest/testdata/config/09_dotdict_format/dotdict_format_config_03.json b/atest/testdata/config/09_dotdict_format/dotdict_format_config_03.json new file mode 100644 index 00000000..609a5258 --- /dev/null +++ b/atest/testdata/config/09_dotdict_format/dotdict_format_config_03.json @@ -0,0 +1,26 @@ +// Copyright 2020-2022 Robert Bosch Car Multimedia GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +{ + "WelcomeString": "Hello... JsonPreprocessor selftest is running now!", + "params": { + "global": { + "[import]" : "./import/dotdict_format_common.json" + } + }, + ${params.global.gPrepro.Float.Param} : 9.999, + ${params.global.gPrepro.Structure.test.Structure} : "Change value", + ${params.global.gPrepro.Structure.general} : "Update value", + "abc" : "This is a multline string\nwith\nhttp://www.google.de\na link inside", + "Target//Name" : "gen3flex//dlt" + } \ No newline at end of file diff --git a/atest/testdata/config/09_dotdict_format/dotdict_format_config_04.json b/atest/testdata/config/09_dotdict_format/dotdict_format_config_04.json new file mode 100644 index 00000000..fff6155d --- /dev/null +++ b/atest/testdata/config/09_dotdict_format/dotdict_format_config_04.json @@ -0,0 +1,25 @@ +// Copyright 2020-2022 Robert Bosch Car Multimedia GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +{ + "WelcomeString": "Hello... JsonPreprocessor selftest is running now!", + "params": { + "global": { + "[import]" : "./import/dotdict_format_common.json" + } + }, + ${params.global.gPrepro.Float.Param} : ${params.global.gPreprol.IntParam}, + ${params.global.gPrepro.Structure.test.Structure} : ${params.global.gPrepro.Structure.general}, + "abc" : "This is a multline string\nwith\nhttp://www.google.de\na link inside", + "Target//Name" : "gen3flex//dlt" + } \ No newline at end of file diff --git a/atest/testdata/config/09_dotdict_format/import/dotdict_format_common.json b/atest/testdata/config/09_dotdict_format/import/dotdict_format_common.json new file mode 100644 index 00000000..2a913c13 --- /dev/null +++ b/atest/testdata/config/09_dotdict_format/import/dotdict_format_common.json @@ -0,0 +1,28 @@ +// Copyright 2020-2022 Robert Bosch Car Multimedia GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +{ + "gPreprol.IntParam" : 1, + + "gPrepro.Float.Param" : 1.332, + + "g.Prepro.String.test" : "This is a string", + + "gPrepro.Structure": { + "test.Structure": "general", + "general": "Structure test" + }, + "scope.Check": "For testing purpose" + } \ No newline at end of file From 642357cd4d4b62106203e9d3ab461c9d7237b6e1 Mon Sep 17 00:00:00 2001 From: mas2hc Date: Wed, 5 Apr 2023 20:42:21 +0700 Subject: [PATCH 3/3] Fixed the issue variable not found --- JsonPreprocessor/CJsonPreprocessor.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index acdd53f5..b6ace24f 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -715,18 +715,5 @@ def __handleStrNoneTrueFalse(objJson): for k, v in oJson.items(): globals().update({k:v}) - # Checking availability of nested parameters before updating and replacing. - for param in self.lNestedParams: - parseNestedParam = self.__nestedParamHandler(param) - tmpParseNestedParam = re.sub('\\${\s*(.*?)\s*}', '\\1', parseNestedParam) - sExec = "value = " + tmpParseNestedParam if isinstance(tmpParseNestedParam, str) else \ - "value = " + str(tmpParseNestedParam) - try: - ldict = {} - exec(sExec, globals(), ldict) - except: - raise Exception(f"The variable '{tmpParseNestedParam}' is not available!") oJson, bNested = self.__updateAndReplaceNestedParam(oJson) - # for k, v in oJson.items(): - # globals().update({k:v}) return oJson \ No newline at end of file