From e8582fa1b1fe552760031651516c4d273fa7d131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Pollersp=C3=B6ck?= Date: Tue, 15 Mar 2022 17:26:30 +0100 Subject: [PATCH 1/4] stabilize path operations, optimize remove_comments, add initial sample for documentation --- JsonPreprocessor/CJsonPreprocessor.py | 120 ++++++++++++++++++-------- sample/json/json_with_comment.json | 16 ++++ sample/json/second_file.json | 15 ++++ sample/sample.py | 12 +++ 4 files changed, 126 insertions(+), 37 deletions(-) create mode 100644 sample/json/json_with_comment.json create mode 100644 sample/json/second_file.json create mode 100644 sample/sample.py diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index e3b2c33c..1b589341 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -48,6 +48,8 @@ import os import json import re +import sys +import platform class CSyntaxType(): python = "python" @@ -70,7 +72,7 @@ class CPythonJSONDecoder(json.JSONDecoder): """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.scan_once = self.custom_scan_once + self.scan_once = self.custom_scan_once def _custom_scan_once(self, string, idx): try: @@ -134,6 +136,50 @@ def __init__(self, syntax=CSyntaxType.json, currentCfg={}): self.currentCfg = currentCfg self.lUpdatedParams = [] + # + # Python struggles with + # - UNC paths + # e.g. \\hi-z4939\ccstg\.... + # - escape sequences in windows paths + # e.g. c:\autotest\tuner \t will be interpreted as tab, the result + # after processing it with an regexp wuld be + # c:\autotest uner + # + # In order to solve this problems any slash will be replaced from backslash + # to slash, only the two UNC backslashes must be kept if contained. + def sNormalizePath(self,sPath): + if sPath.strip()=='': + return '' + + # TML Syntax uses %Name%-syntax to reference an system- or framework + # environment variable. Linux requires ${Name} to do the same. + # Therefore change on Linux systems to ${Name}-syntax to make + # expandvars working here, too. + # This makes same TML code working on both platforms + if platform.system().lower()!="windows": + sPath=re.sub("%(.*?)%","${\\1}",sPath) + + #in a windows system normpath turns all slashes to backslash + #this is unwanted. Therefore turn back after normpath execution. + sNPath=os.path.normpath(os.path.expandvars(sPath.strip())) + #make all backslashes to slash, but mask + #UNC indicator \\ before and restore after. + sNPath=self.__mkslash(sNPath) + + return sNPath + + # make all backslashes to slash, but mask + # UNC indicator \\ before and restore after. + def __mkslash(self,sPath): + if sPath.strip()=='': + return '' + + sNPath=re.sub(r"\\\\",r"#!#!#",sPath.strip()) + sNPath=re.sub(r"\\",r"/",sNPath) + sNPath=re.sub(r"#!#!#",r"\\\\",sNPath) + + return sNPath + ''' Method: __processImportFiles this is custom decorder of object_pairs_hook function. @@ -161,6 +207,14 @@ def __processImportFiles(self, input_data): out_dict[key] = value return out_dict + + def comment_remover(text): + + + + + return + ''' Method: __removeComments loads json config file which allows comments inside Args: @@ -169,41 +223,22 @@ def __processImportFiles(self, input_data): lJsonData: list, list of string data from jsonFile after removing comment(s). ''' def __removeComments(self, jsonFile): - jsonPath = '' - if '/' in jsonFile: - for item in jsonFile.split('/')[:-1]: - jsonPath += item + '/' - else: - for item in jsonFile.split('\\')[:-1]: - jsonPath += item + '\\' - - ''' - Removes comment parts in json file then store in temporary json file - ''' - lJsonData = [] - with open(jsonFile) as fr: - for line in fr: - if re.match('^\s*//', line): - continue - elif '//' in line: - reEx1 = re.search("(\s*{*\s*\'.+\')\s*:\s*(\'.+\'\s*,*)*\s*(.*)", line) - if reEx1 is None: - reEx1 = re.search("(\s*{*\s*\".+\")\s*:\s*(\".+\"\s*,*)*\s*(.*)", line) - if reEx1 is None: - line = re.sub('//.*', '', line) - elif reEx1.group(1) is not None and reEx1.group(2) is not None: - line = reEx1.group(1) + ": " + reEx1.group(2) if reEx1.group(3) is None else \ - reEx1.group(1) + ": " + reEx1.group(2) + re.sub('//.*', '', reEx1.group(3)) - else: - reEx2 = re.search("(\s*{*\s*\'.+\')\s*:\s*(.+,*)\s*//\s*.*", line) - if reEx2 is None: - reEx2 = re.search("(\s*{*\s*\".+\")\s*:\s*(.+,*)\s*(//\s*.*)*", line) - if reEx2 is not None: - line = reEx2.group(1) + ": " + re.sub('//.*', '', reEx2.group(2)) - - lJsonData.append(line) - return lJsonData, jsonPath + def replacer(match): + s = match.group(0) + if s.startswith('/'): + return "" + else: + return s + + file=open(jsonFile,mode='r') + sContent=file.read() + file.close() + + pattern = re.compile(r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE) + sContentCleaned=re.sub(pattern, replacer, sContent) + + return sContentCleaned ''' private __nestedParamHandler: This method handles the nested variable in param names or value @@ -362,13 +397,24 @@ def jsonLoad(self, jFile, masterFile=True): Returns: oJson: dict ''' + jFile=jFile.strip() + + if not re.match("^[a-zA-Z]:",jFile) and not re.match("^[\\/]",jFile): + jFile=self.sNormalizePath(os.path.dirname(sys.argv[0])+"/"+jFile) + + if not(os.path.isfile(jFile)): + raise Exception(f"File '{jFile}' is not existing!") + + (jsonPath,tail)=os.path.split(jFile) + try: - lJsonData, jsonPath = self.__removeComments(jFile) + lJsonData = self.__removeComments(os.path.abspath(jFile)) except Exception as reason: raise Exception("Could not read json configuration file %s due to: %s \n\ Please input 'utf-8' format in Json configuration file only" %(jFile, reason)) currentDir = os.getcwd() + self.lImportedFiles.append(os.path.abspath(jFile)) os.chdir(jsonPath) CJSONDecoder = None @@ -379,7 +425,7 @@ def jsonLoad(self, jFile, masterFile=True): raise Exception('Provided syntax \'%s\' is not supported.' %self.syntax) try: - oJson = json.loads("\n".join(lJsonData), + oJson = json.loads(lJsonData, cls=CJSONDecoder , object_pairs_hook=self.__processImportFiles) except Exception as error: diff --git a/sample/json/json_with_comment.json b/sample/json/json_with_comment.json new file mode 100644 index 00000000..b5c1a79c --- /dev/null +++ b/sample/json/json_with_comment.json @@ -0,0 +1,16 @@ +//Comment1 + +/* a new + multiline comment +*/ + +{ + //Comment2 + "myVar1" : "val1", //Comment3 + /* comment 4 + "myVar2" : "val2 + */ + + "myVar2" : "http://www.google.de", + "[import]" : "./json/second_file.json" +} \ No newline at end of file diff --git a/sample/json/second_file.json b/sample/json/second_file.json new file mode 100644 index 00000000..deb3d8f5 --- /dev/null +++ b/sample/json/second_file.json @@ -0,0 +1,15 @@ +//Comment1 + +/* a new + multiline comment +*/ + +{ + //Comment2 + "second_myVar1" : "val1", //Comment3 + /* comment 4 + "myVar2" : "val2 + */ + + "second_myVar2" : "http://www.amazon.de" +} \ No newline at end of file diff --git a/sample/sample.py b/sample/sample.py new file mode 100644 index 00000000..ba88092c --- /dev/null +++ b/sample/sample.py @@ -0,0 +1,12 @@ +import sys +sys.path.append('D:\B\python-jsonpreprocessor') + +import JsonPreprocessor +from pprint import pprint + +prepro=JsonPreprocessor.CJsonPreprocessor() + +data=prepro.jsonLoad(".\json\json_with_comment.json") + +pprint(data) + From e6a3dfd62f1d31d0111630fccdb2049423e86e2a Mon Sep 17 00:00:00 2001 From: "Pollerspoeck Thomas (CM-CI1/ESY1)" Date: Fri, 18 Mar 2022 16:47:48 +0100 Subject: [PATCH 2/4] maintenance and improvements - add annotations to functions - absolute and relative paths are allowed for main file as well as [import] file. - %envvariable% and {envvariable} syntax are allowed in paths - [import] is now relative to the current file, instead of relative to the base file. This allows shorter paths and it allows to move whole sets of json files to other locations. Only the path to root file of a subset need to be changed in this case. - all docstrings are now in rst syntax and maintained - lot of documenation added - f syntax is use for all exception strings. - unified style of all exception strings. --- JsonPreprocessor/CJsonPreprocessor.py | 320 ++++++++++++++++---------- JsonPreprocessor/__init__.py | 4 +- config/CConfig.py | 2 +- sample/json/json_with_comment.json | 10 +- sample/json/second_file.json | 6 +- sample/json/subfolder/third_file.json | 15 ++ sample/sample.py | 7 +- 7 files changed, 240 insertions(+), 124 deletions(-) create mode 100644 sample/json/subfolder/third_file.json diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index 1b589341..4d0583bd 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -55,26 +55,25 @@ class CSyntaxType(): python = "python" json = "json" -NUMBER_RE = re.compile( +class CPythonJSONDecoder(json.JSONDecoder): + """ +**Method: PythonJSONDecoder** + Add python data types and syntax to json. ``True``, ``False`` and ``None`` will be a accepted as json syntax elements. + +**Args:** + **json.JSONDecoder** (*object*) + Decoder object provided by ``json.loads`` + """ + + NUMBER_RE = re.compile( r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', (re.VERBOSE | re.MULTILINE | re.DOTALL)) -class CPythonJSONDecoder(json.JSONDecoder): - """ Add below python values when scanning json data - - +---------------+-------------------+ - | True | True | - +---------------+-------------------+ - | False | False | - +---------------+-------------------+ - | None | None | - +---------------+-------------------+ - """ - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.scan_once = self.custom_scan_once - def _custom_scan_once(self, string, idx): + def _custom_scan_once(self, string :str, idx: int) -> any: try: nextchar = string[idx] except IndexError: @@ -100,7 +99,7 @@ def _custom_scan_once(self, string, idx): elif nextchar == 'F' and string[idx:idx + 5] == 'False': return False, idx + 5 - m = NUMBER_RE.match(string, idx) + m = CPythonJSONDecoder.NUMBER_RE.match(string, idx) if m is not None: integer, frac, exp = m.groups() if frac or exp: @@ -117,39 +116,107 @@ def _custom_scan_once(self, string, idx): else: raise StopIteration(idx) - def custom_scan_once(self, string, idx): + def custom_scan_once(self, string : str, idx : int) -> any: try: return self._custom_scan_once(string, idx) finally: self.memo.clear() class CJsonPreprocessor(): - ''' - CJsonPreprocessor helps to handle configuration file as json format: - - Allow comment within json file - - Allow import json file within json file - ''' - def __init__(self, syntax=CSyntaxType.json, currentCfg={}): + """ +CJsonPreprocessor extends the syntax of json. + +Features are + +- Allow c/c++-style comments within json files. + + // single line or part of single line and /\* \*/ multline comments are possible + +- Allow to import json files into json files + + ``"[import]" : "relative/absolute path"``, imports another json file to exactly this location. + + ``%envariable%`` and ``${envariable}`` can be used, too. + +- Allow use of variables within json files + + In any place the syntax ``${basenode.subnode. ... nodename}`` allows to reference an already existing variable. + + Example: + + .. code:: json + + { + "basenode" : { + subnode : { + "myparam" : 5 + }, + + }, + + "myVar" : "${basenode.subnode.myparam}" + } + +- Allow python data types ``True``, ``False`` and ``None`` + """ + + def __init__(self, syntax: CSyntaxType = CSyntaxType.json , currentCfg : dict = {}) -> None: + """ +**Method: __init__** + Constructor + +**Args:** + **syntax** (*CSyntaxType*) optional + default: `json` , `python` + If set to `python`, then python data types are allowed as part of json file. + + **currentCfg** (*dict*) optional + Internally used to aggregate imported json files. + """ self.lImportedFiles = [] self.recursive_level = 0 self.syntax = syntax self.currentCfg = currentCfg self.lUpdatedParams = [] - # - # Python struggles with - # - UNC paths - # e.g. \\hi-z4939\ccstg\.... - # - escape sequences in windows paths - # e.g. c:\autotest\tuner \t will be interpreted as tab, the result - # after processing it with an regexp wuld be - # c:\autotest uner - # - # In order to solve this problems any slash will be replaced from backslash - # to slash, only the two UNC backslashes must be kept if contained. - def sNormalizePath(self,sPath): - if sPath.strip()=='': - return '' + + def __sNormalizePath(self, sPath : str) -> str: + """ +**Method: __sNormalizePath** + Python struggles with + + - UNC paths + e.g. ``\\hi-z4939\ccstg\....`` + - escape sequences in windows paths + e.g. ``c:\autotest\tuner \t`` will be interpreted as tab, the result + after processing it with an regexp would be ``c:\autotest uner`` + + In order to solve this problems any slash will be replaced from backslash + to slash, only the two UNC backslashes must be kept if contained. + +**Args:** + **sPath** (*string*) + Absolute or relative path as input. + + Allows environment variables with ``%variable%`` or ``${variable}`` syntax. + +**Returns:** + **sPath** (*string*) + normalized path as string + """ + # make all backslashes to slash, but mask + # UNC indicator \\ before and restore after. + def __mkslash(sPath : str) -> str: + if sPath.strip()=='': + return '' + + sNPath=re.sub(r"\\\\",r"#!#!#",sPath.strip()) + sNPath=re.sub(r"\\",r"/",sNPath) + sNPath=re.sub(r"#!#!#",r"\\\\",sNPath) + + return sNPath + if sPath.strip()=='': + return '' # TML Syntax uses %Name%-syntax to reference an system- or framework # environment variable. Linux requires ${Name} to do the same. @@ -164,65 +231,66 @@ def sNormalizePath(self,sPath): sNPath=os.path.normpath(os.path.expandvars(sPath.strip())) #make all backslashes to slash, but mask #UNC indicator \\ before and restore after. - sNPath=self.__mkslash(sNPath) + sNPath=__mkslash(sNPath) - return sNPath + return sNPath - # make all backslashes to slash, but mask - # UNC indicator \\ before and restore after. - def __mkslash(self,sPath): - if sPath.strip()=='': - return '' - - sNPath=re.sub(r"\\\\",r"#!#!#",sPath.strip()) - sNPath=re.sub(r"\\",r"/",sNPath) - sNPath=re.sub(r"#!#!#",r"\\\\",sNPath) - - return sNPath + def __processImportFiles(self, input_data : dict) -> dict: + ''' +**Method: __processImportFiles** + this is a custom decorder of ``json.loads object_pairs_hook`` function. + + This method helps to import json files which are provided in ``"[import]"`` keyword into the current json file. + +**Args:** + **input_data** (*dict*) + dictionary from json file as input + +**Returns:** + **out_dict** (*dict*) + dictionary as output - ''' - Method: __processImportFiles this is custom decorder of object_pairs_hook function. - This method helps to import json file which is provided in '[import]' keyword into current json file. - Returns: - Dictionary is parsed from json file. - ''' - def __processImportFiles(self, input_data): + dictionary is extended if ``"[import]"`` found and properly imported. + ''' out_dict = {} + for key, value in input_data: if re.match('^\s*\[\s*import\s*\]\s*', key.lower()): abs_path_file = os.path.abspath(value) - + # Use recursive_level and lImportedFiles to avoid cyclic import self.recursive_level = self.recursive_level + 1 # increase recursive level + # length of lImportedFiles should equal to recursive_level self.lImportedFiles = self.lImportedFiles[:self.recursive_level] if abs_path_file in self.lImportedFiles: - raise Exception('Cyclic imported json file \'%s\'' %str(abs_path_file)) + raise Exception(f"Cyclic imported json file '{abs_path_file}'!") - oJsonImport = self.jsonLoad(value, masterFile=False) + oJsonImport = self.jsonLoad(abs_path_file, masterFile=False) out_dict.update(oJsonImport) + self.recursive_level = self.recursive_level - 1 # descrease recursive level else: out_dict[key] = value return out_dict + def __load_and_removeComments(self, jsonFile : str) -> str: + """ +**Method: __load_and_removeComments** + loads a given json file and filters all C/C++ style comments. - def comment_remover(text): +**Args:** + **jsonFile** (*string*) + path (absolute/relative/) file to be processed. + The path can contain windows/linux style environment variables. - + !ATTENTION! This is dangerous - - return - - ''' - Method: __removeComments loads json config file which allows comments inside - Args: - jsonFile: string - Returns: - lJsonData: list, list of string data from jsonFile after removing comment(s). - ''' - def __removeComments(self, jsonFile): +**Returns:** + **sContentCleaned** (*string*) + string version of json file after removing all comments. + """ def replacer(match): s = match.group(0) @@ -240,15 +308,20 @@ def replacer(match): return sContentCleaned - ''' - private __nestedParamHandler: This method handles the nested variable in param names or value - in updated json config file. - Args: - sInputStr: string - param name or value which contains nested variable - Returns: - sStrHandled: string - ''' - def __nestedParamHandler(self, sInputStr): + + def __nestedParamHandler(self, sInputStr : str) -> str: + ''' +**Method: __nestedParamHandler** + This method handles nested variables in parameter names or values. Variable syntax is ${Variable_Name}. + +**Args:** + **sInputStr** (*string*) + Parameter name or value which contains a nested variable. + +**Returns:** + **sStrHandled** (*string*) + String which contains the resolved variable. + ''' #globals().update(currentCfg) referVars = re.findall('(\${\s*.*?\s*})', sInputStr) @@ -266,7 +339,7 @@ def __nestedParamHandler(self, sInputStr): exec(sExec, globals(), ldict) tmpValue = ldict['value'] except: - raise Exception('The variable %s is not available' % (var)) + raise Exception(f"The variable '{var}' is not available!") sInputStr = re.sub('\\' + var, tmpValue, sInputStr) if isinstance(tmpValue, str) else \ re.sub('\\' + var, str(tmpValue), sInputStr) continue @@ -282,7 +355,7 @@ def __nestedParamHandler(self, sInputStr): exec(sExec, globals(), ldict) tmpValue = ldict['value'] except: - raise Exception('The variable %s is not available!!!' % (fullVariable)) + raise Exception("fThe variable '{fullVariable}' is not available!") pattern = re.sub('\[', '\\[', fullVariable) pattern = re.sub('\]', '\\]', pattern) sInputStr = re.sub('\\' + pattern, '\'' + tmpValue + '\'', sInputStr) if isinstance(tmpValue, str) else \ @@ -306,16 +379,22 @@ def __nestedParamHandler(self, sInputStr): sStrHandled = fullVariable return sStrHandled - ''' - private __updateAndReplaceNestedParam: this method replaces all nested params in key and value of Json object - Args: - oJson: dict - currentCfg: dict - Returns: - oJsonOut: dict - ''' - def __updateAndReplaceNestedParam(self, oJson, recursive=False): - + + def __updateAndReplaceNestedParam(self, oJson : dict, recursive : bool = False): + ''' +**Method: __updateAndReplaceNestedParam** + This method replaces all nested parameters in key and value of a json object . + +**Args:** + **oJson** (*dict*) + Input Json object as dictionary. This dictionary will be searched for all ``${variable}`` occurences. + If found it will be replaced with it's current value. + +**Returns:** + **oJsonOut** (*dict*) + Output Json object as dictionary with all variables resolved. + ''' + if bool(self.currentCfg) and not recursive: for k, v in self.currentCfg.items(): globals().update({k:v}) @@ -336,7 +415,7 @@ def __updateAndReplaceNestedParam(self, oJson, recursive=False): try: exec(sExec, globals()) except: - raise Exception("Could not set variable \'%s\' with value \'%s\'" %(k, v)) + raise Exception(f"Could not set variable '{k}' with value '{v}'!") else: tmpJson[k] = v bNested = False @@ -356,7 +435,7 @@ def __updateAndReplaceNestedParam(self, oJson, recursive=False): v = ldict['value'] if v.strip()==valueAfterProcessed else \ v.replace(valueAfterProcessed, str(ldict['value'])) except: - raise Exception('The variable %s is not available!!!' % (tmpValueAfterProcessed)) + raise Exception(f"The variable '{tmpValueAfterProcessed}' is not available!") if bNested: if '[' in k: @@ -364,7 +443,7 @@ def __updateAndReplaceNestedParam(self, oJson, recursive=False): try: exec(sExec, globals()) except: - raise Exception("Could not set variable \'%s\' with value \'%s\'" %(k, v)) + raise Exception(f"Could not set variable '{k}' with value '{v}'!") else: tmpJson[k] = v bNested = False @@ -378,7 +457,7 @@ def __updateAndReplaceNestedParam(self, oJson, recursive=False): try: exec(sExec, globals()) except: - raise Exception("Could not set variable \'%s\' with value \'%s\'" %(k, v)) + raise Exception(f"Could not set variable '{k}' with value '{v}'!") else: tmpJson[k] = v @@ -388,54 +467,63 @@ def __updateAndReplaceNestedParam(self, oJson, recursive=False): return oJson - def jsonLoad(self, jFile, masterFile=True): + + def jsonLoad(self, jFile : str, masterFile : bool = True): ''' - Method: jsonLoad loads the json file then parses to dict object - - Args: - jFile: string, json file input - Returns: - oJson: dict +**Method: jsonLoad** + This function is the entry point of JsonPreprocessor. + + It loads the json file, preprocesses it and returns the preprocessed result as data structure. + +**Args:** + **jFile** (*string*) + relative/absolute path to main json file. + + ``%envvariable%`` and ``${envvariable}`` can be used, too in order to access environment variables. + +**Returns:** + **oJson** (*dict*) + preprocessed json file(s) as data structure ''' jFile=jFile.strip() if not re.match("^[a-zA-Z]:",jFile) and not re.match("^[\\/]",jFile): - jFile=self.sNormalizePath(os.path.dirname(sys.argv[0])+"/"+jFile) + jFile=self.__sNormalizePath(os.path.dirname(sys.argv[0])+"/"+jFile) if not(os.path.isfile(jFile)): - raise Exception(f"File '{jFile}' is not existing!") + raise Exception(f"File '{jFile}' is not existing!") - (jsonPath,tail)=os.path.split(jFile) + self.lImportedFiles.append(os.path.abspath(jFile)) + (jsonPath,tail)=os.path.split(jFile) try: - lJsonData = self.__removeComments(os.path.abspath(jFile)) + sJsonData= self.__load_and_removeComments(os.path.abspath(jFile)) except Exception as reason: - raise Exception("Could not read json configuration file %s due to: %s \n\ - Please input 'utf-8' format in Json configuration file only" %(jFile, reason)) + raise Exception(f"Could not read json file '{jFile}' due to: '{reason}'!") + currentDir = os.getcwd() - - self.lImportedFiles.append(os.path.abspath(jFile)) os.chdir(jsonPath) + CJSONDecoder = None if self.syntax != CSyntaxType.json: if self.syntax == CSyntaxType.python: CJSONDecoder = CPythonJSONDecoder else: - raise Exception('Provided syntax \'%s\' is not supported.' %self.syntax) + raise Exception(f"Provided syntax '{self.syntax}' is not supported.") try: - oJson = json.loads(lJsonData, + oJson = json.loads(sJsonData, cls=CJSONDecoder , object_pairs_hook=self.__processImportFiles) except Exception as error: - raise Exception("JSON configuration file '%s': %s" %(jFile, error)) + raise Exception(f"json file '{jFile}': '{error}'") os.chdir(currentDir) + if masterFile: for k, v in oJson.items(): globals().update({k:v}) oJson = self.__updateAndReplaceNestedParam(oJson) - # oJson['JsonPath'] = jsonPath # is JsonPath required? return oJson \ No newline at end of file diff --git a/JsonPreprocessor/__init__.py b/JsonPreprocessor/__init__.py index 8043fb51..d897a8bb 100644 --- a/JsonPreprocessor/__init__.py +++ b/JsonPreprocessor/__init__.py @@ -11,4 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from .CJsonPreprocessor import CJsonPreprocessor \ No newline at end of file +from .CJsonPreprocessor import CJsonPreprocessor +from .CJsonPreprocessor import CSyntaxType + diff --git a/config/CConfig.py b/config/CConfig.py index 19b5e276..e040ea49 100644 --- a/config/CConfig.py +++ b/config/CConfig.py @@ -86,7 +86,7 @@ def __init__(self, sReferencePath="."): # 1. basic setup stuff self.__dictConfig['sPackageName'] = "JsonPreprocessor" - self.__dictConfig['sVersion'] = "0.1.0" + self.__dictConfig['sVersion'] = "0.2.0" self.__dictConfig['sAuthor'] = "Mai Dinh Nam Son" self.__dictConfig['sAuthorEMail'] = "son.maidinhnam@vn.bosch.com" self.__dictConfig['sDescription'] = "This package provides a preprocessor for json files" diff --git a/sample/json/json_with_comment.json b/sample/json/json_with_comment.json index b5c1a79c..f5d07c28 100644 --- a/sample/json/json_with_comment.json +++ b/sample/json/json_with_comment.json @@ -12,5 +12,13 @@ */ "myVar2" : "http://www.google.de", - "[import]" : "./json/second_file.json" + "myVar3" : "${myVar1}", + "abc" : + { + "[import]" : "./second_file.json" + }, + "def" : + { + "[import]" : "./subfolder/third_file.json" + } } \ No newline at end of file diff --git a/sample/json/second_file.json b/sample/json/second_file.json index deb3d8f5..41430627 100644 --- a/sample/json/second_file.json +++ b/sample/json/second_file.json @@ -6,10 +6,10 @@ { //Comment2 - "second_myVar1" : "val1", //Comment3 + "second_myVar1" : True, //Comment3 /* comment 4 "myVar2" : "val2 */ - - "second_myVar2" : "http://www.amazon.de" + "[import]" : "./subfolder/third_file.json", + "second_myVar2" : "http://www.ebay.de" } \ No newline at end of file diff --git a/sample/json/subfolder/third_file.json b/sample/json/subfolder/third_file.json new file mode 100644 index 00000000..e6321fc9 --- /dev/null +++ b/sample/json/subfolder/third_file.json @@ -0,0 +1,15 @@ +//Comment1 + +/* a new + multiline comment +*/ + +{ + //Comment2 + "third_myVar1" : "val1", //Comment3 + /* comment 4 + "myVar2" : "val2 + */ + + "third_myVar2" : "http://www.amazon.de" +} \ No newline at end of file diff --git a/sample/sample.py b/sample/sample.py index ba88092c..5924e7e1 100644 --- a/sample/sample.py +++ b/sample/sample.py @@ -1,10 +1,13 @@ import sys + sys.path.append('D:\B\python-jsonpreprocessor') -import JsonPreprocessor +from JsonPreprocessor import CJsonPreprocessor +from JsonPreprocessor import CSyntaxType + from pprint import pprint -prepro=JsonPreprocessor.CJsonPreprocessor() +prepro=CJsonPreprocessor(syntax=CSyntaxType.python) data=prepro.jsonLoad(".\json\json_with_comment.json") From bbab8c8a1457b903dc891282c44ea70fbdc0bfe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Pollersp=C3=B6ck?= Date: Thu, 24 Mar 2022 17:28:53 +0100 Subject: [PATCH 3/4] extend samples --- sample/001_sample_comments.py | 37 ++++++++++++++++++++++ sample/003_sample_import.py | 37 ++++++++++++++++++++++ sample/005_sample_python_syntax.py | 40 ++++++++++++++++++++++++ sample/007_sample_use_of_variables.py | 37 ++++++++++++++++++++++ sample/json/json_with_comment.json | 32 ++++++++++++------- sample/json/json_with_import.json | 38 ++++++++++++++++++++++ sample/json/json_with_python_syntax.json | 24 ++++++++++++++ sample/json/json_with_variables.json | 31 ++++++++++++++++++ sample/json/second_file.json | 17 ++++------ sample/json/subfolder/third_file.json | 13 +------- sample/sample.py | 15 --------- 11 files changed, 272 insertions(+), 49 deletions(-) create mode 100644 sample/001_sample_comments.py create mode 100644 sample/003_sample_import.py create mode 100644 sample/005_sample_python_syntax.py create mode 100644 sample/007_sample_use_of_variables.py create mode 100644 sample/json/json_with_import.json create mode 100644 sample/json/json_with_python_syntax.json create mode 100644 sample/json/json_with_variables.json delete mode 100644 sample/sample.py diff --git a/sample/001_sample_comments.py b/sample/001_sample_comments.py new file mode 100644 index 00000000..93018399 --- /dev/null +++ b/sample/001_sample_comments.py @@ -0,0 +1,37 @@ +# Copyright 2020-2022 Robert Bosch Car Multimedia GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# + +## This example also shows how to use comments +## in json files. + +import sys + +sys.path.append('D:\B\python-jsonpreprocessor') + +from JsonPreprocessor import CJsonPreprocessor + +from pprint import pprint + +prepro=CJsonPreprocessor() + +# you can load the base json file with +# - relative path to your python program +# - absolute path +# - paths containting environment variables by means of +# %envvariable% syntax +data=prepro.jsonLoad(".\json\json_with_comment.json") + +pprint(data) + diff --git a/sample/003_sample_import.py b/sample/003_sample_import.py new file mode 100644 index 00000000..71aa1c94 --- /dev/null +++ b/sample/003_sample_import.py @@ -0,0 +1,37 @@ +# Copyright 2020-2022 Robert Bosch Car Multimedia GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# + +## This example also shows how to import json files +## into json files. + +import sys + +sys.path.append('D:\B\python-jsonpreprocessor') + +from JsonPreprocessor import CJsonPreprocessor + +from pprint import pprint + +prepro=CJsonPreprocessor() + +# you can load the base json file with +# - relative path to your python program +# - absolute path +# - paths containting environment variables by means of +# %envvariable% syntax +data=prepro.jsonLoad(".\json\json_with_import.json") + +pprint(data) + diff --git a/sample/005_sample_python_syntax.py b/sample/005_sample_python_syntax.py new file mode 100644 index 00000000..48459831 --- /dev/null +++ b/sample/005_sample_python_syntax.py @@ -0,0 +1,40 @@ +# Copyright 2020-2022 Robert Bosch Car Multimedia GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# + +## This example also shows how to use python-syntax +## as part of json files. + +import sys + +sys.path.append('D:\B\python-jsonpreprocessor') + +from JsonPreprocessor import CJsonPreprocessor +from JsonPreprocessor import CSyntaxType + +from pprint import pprint + +# CSyntaxType.python activates recognition of python +# syntax as part of json files +prepro=CJsonPreprocessor(syntax=CSyntaxType.python) + +# you can load the base json file with +# - relative path to your python program +# - absolute path +# - paths containting environment variables by means of +# %envvariable% syntax +data=prepro.jsonLoad(".\json\json_with_python_syntax.json") + +pprint(data) + diff --git a/sample/007_sample_use_of_variables.py b/sample/007_sample_use_of_variables.py new file mode 100644 index 00000000..e7b4023a --- /dev/null +++ b/sample/007_sample_use_of_variables.py @@ -0,0 +1,37 @@ +# Copyright 2020-2022 Robert Bosch Car Multimedia GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# + +## This example also shows how to use comments +## in json files. + +import sys + +sys.path.append('D:\B\python-jsonpreprocessor') + +from JsonPreprocessor import CJsonPreprocessor + +from pprint import pprint + +prepro=CJsonPreprocessor() + +# you can load the base json file with +# - relative path to your python program +# - absolute path +# - paths containting environment variables by means of +# %envvariable% syntax +data=prepro.jsonLoad(".\json\json_with_variables.json") + +pprint(data) + diff --git a/sample/json/json_with_comment.json b/sample/json/json_with_comment.json index f5d07c28..a27d9df6 100644 --- a/sample/json/json_with_comment.json +++ b/sample/json/json_with_comment.json @@ -1,24 +1,34 @@ -//Comment1 +// You can write anywhere one line comments -/* a new - multiline comment +/* +This way you can +write anywhere multiline +comments */ { - //Comment2 - "myVar1" : "val1", //Comment3 - /* comment 4 - "myVar2" : "val2 + /* + This way you can + write anywhere multiline + comments */ + // You can write anywhere one line comments + "myVar1" : "val1", + "myVar2" : "http://www.google.de", - "myVar3" : "${myVar1}", - "abc" : + "myVar3" : "val3", // You can write anywhere one line comments + "abc" : // You can write anywhere one line comments { - "[import]" : "./second_file.json" + /* + This way you can + write anywhere multiline + comments + */ + "myVar4" : "val4" }, "def" : { - "[import]" : "./subfolder/third_file.json" + "myVar5" : "val5" // You can write anywhere one line comments } } \ No newline at end of file diff --git a/sample/json/json_with_import.json b/sample/json/json_with_import.json new file mode 100644 index 00000000..c8e1fdc5 --- /dev/null +++ b/sample/json/json_with_import.json @@ -0,0 +1,38 @@ +/* +imports of json files are done by means of the +[import] keyword. + +All imported data will be replace the node which +has the [import] keyword. + +[import] can be done at any location in your json +data structure. This allows to structure big json files +into any architecture you require. +*/ +{ + "myVar1" : "val1", + + "myVar2" : "http://www.google.de", + "abc" : + { + /* + import of json files should be always a relative path + to the current file. This allows later to shift whole + json directories to other places without braking the + internal strucutre. + + All imported data will be located in this case below + node "abc" and replace the [import] node. + */ + "[import]" : "./second_file.json" + }, + "def" : + { + //All imported data will be located in this case below + //node "def" and replace the [import] node. + //this file is imported in this example two times as + //re-usable data structure. + //one time here, a second time in "second_file.json" + "[import]" : "./subfolder/third_file.json" + } +} \ No newline at end of file diff --git a/sample/json/json_with_python_syntax.json b/sample/json/json_with_python_syntax.json new file mode 100644 index 00000000..a14f81d6 --- /dev/null +++ b/sample/json/json_with_python_syntax.json @@ -0,0 +1,24 @@ +{ + //instead of `true` you can use the python-syntax `True` + "myVar1" : True, + "myVar1_" : true, + + //instead of `frue` you can use the python-syntax `False` + "myVar2" : False, + "myVar2_" : false, + + //instead of `null` you can use the python-syntax `None` + "myVar3" : None, + "myVar3_" : null, + + "abc" : + { + //this is a string -> there is no change + "myVar4" : "True" + }, + "def" : + { + //this is a string -> there is no change + "myVar5" : "False" + } +} \ No newline at end of file diff --git a/sample/json/json_with_variables.json b/sample/json/json_with_variables.json new file mode 100644 index 00000000..68ba0844 --- /dev/null +++ b/sample/json/json_with_variables.json @@ -0,0 +1,31 @@ +/* +every node can be used by means of ${node} syntax +as variable once it was defined. +This works also with [import] of json data. As per +order the [import ] appears in the code the nodes +can be referenced by means of ${syntax} as variable. + */ + +{ + // create some initial data + "myVar1" : "val1", + "myVar2" : "val2", + "abc" : { + "arList" : ["a","b","c"] + }, + + //give myVar3 the value of "myVar1" + "myVar3" : "${myVar1}", + + //give myVar4 the value of ${abc}['arList'][1] + "myVar4" : "${abc}['arList'][1]", + + //give myVar4 the value of ${abc}['arList'] + "myVar5" : "${abc}['arList']", + + //add a new entry to node "abc" + "${abc}['new_entry']" : "new_entry", + + //give myVar6 the value new value of node "abc" + "myVar6" : "${abc}" +} \ No newline at end of file diff --git a/sample/json/second_file.json b/sample/json/second_file.json index 41430627..c2a9deb5 100644 --- a/sample/json/second_file.json +++ b/sample/json/second_file.json @@ -1,15 +1,10 @@ -//Comment1 - -/* a new - multiline comment -*/ - { - //Comment2 - "second_myVar1" : True, //Comment3 - /* comment 4 - "myVar2" : "val2 - */ + "second_myVar1" : true, + + //All imported data will be located in this case below + //the root node of this file. + //The import is done again relative to this file. "[import]" : "./subfolder/third_file.json", + "second_myVar2" : "http://www.ebay.de" } \ No newline at end of file diff --git a/sample/json/subfolder/third_file.json b/sample/json/subfolder/third_file.json index e6321fc9..bc27482b 100644 --- a/sample/json/subfolder/third_file.json +++ b/sample/json/subfolder/third_file.json @@ -1,15 +1,4 @@ -//Comment1 - -/* a new - multiline comment -*/ - { - //Comment2 - "third_myVar1" : "val1", //Comment3 - /* comment 4 - "myVar2" : "val2 - */ - + "third_myVar1" : "val1", "third_myVar2" : "http://www.amazon.de" } \ No newline at end of file diff --git a/sample/sample.py b/sample/sample.py deleted file mode 100644 index 5924e7e1..00000000 --- a/sample/sample.py +++ /dev/null @@ -1,15 +0,0 @@ -import sys - -sys.path.append('D:\B\python-jsonpreprocessor') - -from JsonPreprocessor import CJsonPreprocessor -from JsonPreprocessor import CSyntaxType - -from pprint import pprint - -prepro=CJsonPreprocessor(syntax=CSyntaxType.python) - -data=prepro.jsonLoad(".\json\json_with_comment.json") - -pprint(data) - From 9b0d61bdb4dffb5faa03d8deefd6b95a4e3d84bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Pollersp=C3=B6ck?= Date: Fri, 25 Mar 2022 13:53:39 +0100 Subject: [PATCH 4/4] add licenze to sample files --- sample/json/json_with_comment.json | 14 ++++++++++++++ sample/json/json_with_import.json | 14 ++++++++++++++ sample/json/json_with_python_syntax.json | 13 +++++++++++++ sample/json/json_with_variables.json | 14 ++++++++++++++ sample/json/second_file.json | 13 +++++++++++++ sample/json/subfolder/third_file.json | 13 +++++++++++++ 6 files changed, 81 insertions(+) diff --git a/sample/json/json_with_comment.json b/sample/json/json_with_comment.json index a27d9df6..97ba1a01 100644 --- a/sample/json/json_with_comment.json +++ b/sample/json/json_with_comment.json @@ -1,3 +1,17 @@ +/*Copyright 2020-2022 Robert Bosch Car Multimedia GmbH + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + // You can write anywhere one line comments /* diff --git a/sample/json/json_with_import.json b/sample/json/json_with_import.json index c8e1fdc5..60bdc833 100644 --- a/sample/json/json_with_import.json +++ b/sample/json/json_with_import.json @@ -1,3 +1,17 @@ +/*Copyright 2020-2022 Robert Bosch Car Multimedia GmbH + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + /* imports of json files are done by means of the [import] keyword. diff --git a/sample/json/json_with_python_syntax.json b/sample/json/json_with_python_syntax.json index a14f81d6..40522e76 100644 --- a/sample/json/json_with_python_syntax.json +++ b/sample/json/json_with_python_syntax.json @@ -1,3 +1,16 @@ +/*Copyright 2020-2022 Robert Bosch Car Multimedia GmbH + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ { //instead of `true` you can use the python-syntax `True` "myVar1" : True, diff --git a/sample/json/json_with_variables.json b/sample/json/json_with_variables.json index 68ba0844..59439a39 100644 --- a/sample/json/json_with_variables.json +++ b/sample/json/json_with_variables.json @@ -1,3 +1,17 @@ +/*Copyright 2020-2022 Robert Bosch Car Multimedia GmbH + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + /* every node can be used by means of ${node} syntax as variable once it was defined. diff --git a/sample/json/second_file.json b/sample/json/second_file.json index c2a9deb5..e112566b 100644 --- a/sample/json/second_file.json +++ b/sample/json/second_file.json @@ -1,3 +1,16 @@ +/*Copyright 2020-2022 Robert Bosch Car Multimedia GmbH + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ { "second_myVar1" : true, diff --git a/sample/json/subfolder/third_file.json b/sample/json/subfolder/third_file.json index bc27482b..925ee1dd 100644 --- a/sample/json/subfolder/third_file.json +++ b/sample/json/subfolder/third_file.json @@ -1,3 +1,16 @@ +/*Copyright 2020-2022 Robert Bosch Car Multimedia GmbH + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ { "third_myVar1" : "val1", "third_myVar2" : "http://www.amazon.de"