From a59a3431f02cb8e3f219abce4e1a6e982ec1cc06 Mon Sep 17 00:00:00 2001 From: mas2hc Date: Tue, 13 Aug 2024 09:55:12 +0700 Subject: [PATCH 01/11] Update interface descriptions of jsonLoad and jsonLoads methods --- JsonPreprocessor/CJsonPreprocessor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index 322a2894..251584fb 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -1384,7 +1384,7 @@ def jsonLoad(self, jFile : str, masterFile : bool = True): * ``masterFile`` - / *Condition*: required / *Type*: bool / + / *Condition*: optional / *Type*: bool / Identifies the entry level when loading JSONP file in comparison with imported files levels. Default value is True @@ -1427,13 +1427,13 @@ def jsonLoads(self, sJsonpContent : str, referenceDir : str = '', firstLevel : b * ``referenceDir`` - / *Condition*: required / *Type*: str / + / *Condition*: optional / *Type*: str / A reference path for loading imported files. * ``firstLevel`` - / *Condition*: required / *Type*: bool / + / *Condition*: optional / *Type*: bool / Identifies the entry level when loading JSONP content in comparison with imported files levels. From ebe6b92180b2061d5b2a84f1a47d9f7caca7431e Mon Sep 17 00:00:00 2001 From: mas2hc Date: Tue, 13 Aug 2024 10:11:29 +0700 Subject: [PATCH 02/11] Update error message in naming convention check --- JsonPreprocessor/CJsonPreprocessor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index 251584fb..7273a7f9 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -1335,10 +1335,10 @@ def __keyNameValidation(self, sInput): errorMsg = f"A substitution in key names is not allowed! Please update the key name {sInput}" elif '${' not in sInput and not re.match(r'^\s*"\[\s*import\s*\]"\s*$', sInput.lower()): if re.match(r'^[\s"]*[\+\-\*:@]+.*$', sInput): - errorMsg = f"Invalid key name: {sInput}. Key names have to start with a character or digit." + errorMsg = f"Invalid key name: {sInput}. Key names have to start with a character, digit or underscore." elif checkPattern.search(sInput): errorMsg = f"Invalid key name: {sInput}. Key names must not contain these special characters \"!#$%^&()=[]{{}}|;',?`~\" \ -and have to start with a character or digit." +and have to start with a character, digit or underscore." elif re.search(r'\${[^}]*}', sInput): if re.search(r'\[\s*\]', sInput): errorMsg = f"Invalid key name: {sInput}. A pair of square brackets is empty!!!" @@ -1351,14 +1351,14 @@ def __keyNameValidation(self, sInput): errorMsg = f"Invalid key name: {sInput}. A pair of curly brackets is empty!!!" break elif re.match(r'^[\+\-\*]+.*$', param[1]): - errorMsg = f"Invalid key name: {sInput}. Key names have to start with a character or digit." + errorMsg = f"Invalid key name: {sInput}. Key names have to start with a character, digit or underscore." break elif re.search(r'^.+\[.+\]$', param[1].strip()): errorMsg = f"Invalid syntax: Found index or sub-element inside curly brackets in the parameter '{sInput}'" break elif checkPattern.search(param[1]): errorMsg = f"Invalid key name: '{param[1]}' in {sInput}. Key names must not contain these special characters \"!#$%^&()=[]{{}}|;',?`~\" \ -and have to start with a character or digit." +and have to start with a character, digit or underscore." break else: nestedParam = param[0] From 9f7d158157a671577ed5292aa7c849e52169f109 Mon Sep 17 00:00:00 2001 From: mas2hc Date: Tue, 13 Aug 2024 10:50:47 +0700 Subject: [PATCH 03/11] Optimized key names naming convention check --- JsonPreprocessor/CJsonPreprocessor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index 7273a7f9..2176466f 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -185,7 +185,7 @@ def __init__(self, syntax: CSyntaxType = CSyntaxType.python , currentCfg : dict import builtins import keyword self.lDataTypes = [name for name, value in vars(builtins).items() if isinstance(value, type)] - self.specialCharacters = r'!@#$%^&()=[]{}|;:\'",?`~/' + self.specialCharacters = r'!#$%^&()=[]{{}}|;\',?`~' self.lDataTypes.append(keyword.kwlist) self.jsonPath = None self.masterFile = None @@ -1225,7 +1225,7 @@ def __checkNestedParam(self, sInput : str, bKey=False, bCheckKeyName=False) -> b while sTmpInput.count("${") > 1: lParams = re.findall(r'\${([^\$}]*)}', sTmpInput) for param in lParams: - if param.strip()=='' or re.search(r'[!@#\$%\^&\*\(\)=\[\]{}|;:\s\+\'",<>?/`~]', param) or \ + if param.strip()=='' or re.search(re.escape(self.specialCharacters), param) or \ re.match(r'^\s*\-+.*\s*$', param) or re.match(r'^\s*[^\-]*\-+\s*$', param): bSpecialCharInParam = True break @@ -1328,7 +1328,7 @@ def __keyNameValidation(self, sInput): *No return value* """ - checkPattern = re.compile(r'[!#$%^&\(\)=\[\]{}\|;\',?`~]') + checkPattern = re.compile(re.escape(self.specialCharacters)) errorMsg = '' if CNameMangling.STRINGCONVERT.value in sInput: sInput = sInput.replace(CNameMangling.STRINGCONVERT.value, '') @@ -1337,7 +1337,7 @@ def __keyNameValidation(self, sInput): if re.match(r'^[\s"]*[\+\-\*:@]+.*$', sInput): errorMsg = f"Invalid key name: {sInput}. Key names have to start with a character, digit or underscore." elif checkPattern.search(sInput): - errorMsg = f"Invalid key name: {sInput}. Key names must not contain these special characters \"!#$%^&()=[]{{}}|;',?`~\" \ + errorMsg = f"Invalid key name: {sInput}. Key names must not contain these special characters \"{self.specialCharacters}\" \ and have to start with a character, digit or underscore." elif re.search(r'\${[^}]*}', sInput): if re.search(r'\[\s*\]', sInput): @@ -1357,7 +1357,7 @@ def __keyNameValidation(self, sInput): errorMsg = f"Invalid syntax: Found index or sub-element inside curly brackets in the parameter '{sInput}'" break elif checkPattern.search(param[1]): - errorMsg = f"Invalid key name: '{param[1]}' in {sInput}. Key names must not contain these special characters \"!#$%^&()=[]{{}}|;',?`~\" \ + errorMsg = f"Invalid key name: '{param[1]}' in {sInput}. Key names must not contain these special characters \"{self.specialCharacters}\" \ and have to start with a character, digit or underscore." break else: From db3b3454c44c5112516854008eeef7b190227bd3 Mon Sep 17 00:00:00 2001 From: mas2hc Date: Thu, 15 Aug 2024 16:25:18 +0700 Subject: [PATCH 04/11] Enhancement 321 - Updated regex pattern --- JsonPreprocessor/CJsonPreprocessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index 2176466f..e3e896c0 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -1537,9 +1537,9 @@ def __handleLastElement(sInput : str) -> str: if "${" in line: curLine = line - while re.search(r'\${([^}]+)}', line): + while re.search(r'\${([^}]*)}', line): tmpLine = line - param = re.search(r'\${([^}\$]+)}', line) + param = re.search(r'\${([^}\$]*)}', line) if param is not None: self.__keyNameValidation(param[0]) if ':' in param[1]: From 536f5ad66d9587596c142ea26d85ae33ea638fbd Mon Sep 17 00:00:00 2001 From: mas2hc Date: Thu, 15 Aug 2024 21:33:49 +0700 Subject: [PATCH 05/11] Enhancement 328 - Added new error message --- JsonPreprocessor/CJsonPreprocessor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index e3e896c0..7171bc71 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -577,6 +577,10 @@ def __getNestedValue(sNestedParam : str): varPattern = re.escape(var[0]) if re.search(r"\[['\s]*" + varPattern + r"['\s]*\]", sInputStr): if re.search(r"\[\s*'\s*" + varPattern + r"\s*'\s*\]", sInputStr): + if (isinstance(tmpValue, list) or isinstance(tmpValue, dict)): + self.__reset() + raise Exception(f"The substitution of parameter '{sVar.replace('$$', '$')}' inside \ +the expression '{sNestedParam}' is not supported! Composite data types like lists and dictionaries cannot be substituted as strings.") sInputStr = re.sub(r"\[\s*'\s*" + varPattern + r"\s*'\s*\]", "['" + str(tmpValue) + "']", sInputStr) elif isinstance(tmpValue, str): sInputStr = re.sub(r"\[['\s]*" + varPattern + r"['\s]*\]", "['" + tmpValue + "']", sInputStr) From e5337799b2e1c67a0e5bef2b224978dc570077bc Mon Sep 17 00:00:00 2001 From: mas2hc Date: Thu, 22 Aug 2024 10:33:00 +0700 Subject: [PATCH 06/11] Enhancement 285 - Parameter name interpreted as regular expression --- JsonPreprocessor/CJsonPreprocessor.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index 7171bc71..5d274861 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -1338,7 +1338,7 @@ def __keyNameValidation(self, sInput): sInput = sInput.replace(CNameMangling.STRINGCONVERT.value, '') errorMsg = f"A substitution in key names is not allowed! Please update the key name {sInput}" elif '${' not in sInput and not re.match(r'^\s*"\[\s*import\s*\]"\s*$', sInput.lower()): - if re.match(r'^[\s"]*[\+\-\*:@]+.*$', sInput): + if re.match(r'^[\s"]*[\+\-\*:@' + re.escape(self.specialCharacters) + ']+.*$', sInput): errorMsg = f"Invalid key name: {sInput}. Key names have to start with a character, digit or underscore." elif checkPattern.search(sInput): errorMsg = f"Invalid key name: {sInput}. Key names must not contain these special characters \"{self.specialCharacters}\" \ @@ -1354,7 +1354,7 @@ def __keyNameValidation(self, sInput): if param[1].strip() == '': errorMsg = f"Invalid key name: {sInput}. A pair of curly brackets is empty!!!" break - elif re.match(r'^[\+\-\*]+.*$', param[1]): + elif re.match(r'^[\+\-\*:@' + re.escape(self.specialCharacters) + ']+.*$', param[1]): errorMsg = f"Invalid key name: {sInput}. Key names have to start with a character, digit or underscore." break elif re.search(r'^.+\[.+\]$', param[1].strip()): @@ -1530,6 +1530,7 @@ def __handleLastElement(sInput : str) -> str: else: sJsonData = sJsonpContent sJsonDataUpdated = "" + lNestedParams = [] for line in sJsonData.splitlines(): if line == '' or line.isspace(): continue @@ -1545,7 +1546,7 @@ def __handleLastElement(sInput : str) -> str: tmpLine = line param = re.search(r'\${([^}\$]*)}', line) if param is not None: - self.__keyNameValidation(param[0]) + lNestedParams.append(param[0]) if ':' in param[1]: tmpList03.append(param[1]) tmpPattern = re.escape(param[1]) @@ -1651,6 +1652,8 @@ def __handleLastElement(sInput : str) -> str: for key in lKeyName: keyDecode = bytes(key, 'utf-8').decode('unicode_escape') self.__keyNameValidation(keyDecode) + for param in lNestedParams: + self.__keyNameValidation(param) CJSONDecoder = None if self.syntax != CSyntaxType.json: if self.syntax == CSyntaxType.python: From edfc159183e8cab835c65d5faef48cd89058c4c6 Mon Sep 17 00:00:00 2001 From: mas2hc Date: Thu, 22 Aug 2024 14:37:01 +0700 Subject: [PATCH 07/11] Enhancement 290 - Line breaks and string conversions --- JsonPreprocessor/CJsonPreprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index 5d274861..ce0c8fb2 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -1538,7 +1538,7 @@ def __handleLastElement(sInput : str) -> str: listDummy = shlex.split(line) except Exception as error: self.__reset() - raise Exception(f"\n{str(error)} in line: '{line}'") + raise Exception(f"{error} in line: '{line}'") if "${" in line: curLine = line From 157564c2102adfdf0badef7cc8322f1a70125f17 Mon Sep 17 00:00:00 2001 From: mas2hc Date: Thu, 22 Aug 2024 15:35:24 +0700 Subject: [PATCH 08/11] Enhancement 312 - list index out of range --- JsonPreprocessor/CJsonPreprocessor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index ce0c8fb2..a4fb9246 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -1463,9 +1463,10 @@ def __handleDuplicatedKey(dInput : dict) -> dict: if CNameMangling.DUPLICATEDKEY_01.value in k: origK = re.sub(CNameMangling.DUPLICATEDKEY_01.value + "\d+\s*$", "", k) dInput[k] = dictValues[origK].pop(1) - if isinstance(v, list) and v[0]==CNameMangling.DUPLICATEDKEY_01.value: - v = v[-1] - dInput[k] = v + if isinstance(v, list): + if len(v)>0 and v[0]==CNameMangling.DUPLICATEDKEY_01.value: + v = v[-1] + dInput[k] = v if isinstance(v, dict): dInput[k] = __handleDuplicatedKey(v) del tmpDict From 20959676274d7c333c0363b8b7403a43338f535f Mon Sep 17 00:00:00 2001 From: qth2hi Date: Thu, 22 Aug 2024 14:12:57 +0200 Subject: [PATCH 09/11] Fixed releaser mail info in file release_items_JsonPreprocessor.json --- .../release_items_JsonPreprocessor.json | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/config/robotframework_aio/release_items_JsonPreprocessor.json b/config/robotframework_aio/release_items_JsonPreprocessor.json index 95c1bf2d..75ae0fea 100644 --- a/config/robotframework_aio/release_items_JsonPreprocessor.json +++ b/config/robotframework_aio/release_items_JsonPreprocessor.json @@ -116,7 +116,7 @@ In opposite to previous versions of the **JsonPreprocessor**, the creation of new keys based on parameter (dynamic key names), **is not supported an more**! - Example: + **Example:** | ``\u007b`` | ``\"param1\" : \"ABC\",`` @@ -126,27 +126,25 @@ This code previously created a new parameter ``\"XYZ\"`` with value ``1``. Now an error message will be raised. -* Improved error messages +* Improved error handling and error messages * Added ``jsonLoads`` method that allows users to directly parse JSONP content from strings **Example:** - | ``jsonpStr = \u007b\"A\" : 1,\n \"[import]\" : \"./imported_file.jsonp\", // relative path of imported file \n \"B\" : 2\u007d`` + | ``jsonpStr = \"\u007b\\\"A\\\" : 1, \\\"B\\\" : 2\u007d\"`` | ``json_preprocessor = CJsonPreprocessor()`` - | ``json_preprocessor.jsonLoads(jsonpStr, referenceDir=\"path_to_reference_dir\")`` + | ``retValues = json_preprocessor.jsonLoads(jsonpStr)`` -* Added a naming convention check for key names within .jsonp files processed by the JsonPreprocessor +* Added a naming convention check for key names within JSONP content processed by the **JsonPreprocessor** - Key names have to start with a character, digit, or underscore and must not contain these special characters !#$%^&()=[]{}|;',?`~ + Key names have to start with a character, digit, or underscore and must not contain these special characters ``!#$%^&()=[]{}|;',?`~`` **Example:** - Valid key names could be \"abcParam\", \"01_Param\", \"__param+1\", \"param-1\", \"abc@jpp.com\", etc. - - Invalid key names could be \"+param01\", \"param$01\", \"abc#Param\", etc. + Valid key names are: ``\"abcParam\"``, ``\"01_Param\"``, ``\"__param+1\"``, ``\"param-1\"``, ``\"abc@jpp.com\"``, ... -* Error handling deviation improvement. + Invalid key names are: ``\"+param01\"``, ``\"param$01\"``, ``\"abc#Param\"``, ... " ] } From 0583c1f0af14731698b541fd44bab3b3c9f60a5f Mon Sep 17 00:00:00 2001 From: mas2hc Date: Mon, 26 Aug 2024 16:10:59 +0700 Subject: [PATCH 10/11] Enhancement 316 - Code improvement in jsonLoad & jsonLoads --- JsonPreprocessor/CJsonPreprocessor.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index a4fb9246..02e1acf0 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -281,7 +281,7 @@ def __processImportFiles(self, input_data : dict) -> dict: # self.__reset() raise Exception(f"Cyclic imported json file '{abs_path_file}'!") - oJsonImport = self.jsonLoad(abs_path_file, masterFile=False) + oJsonImport = self.jsonLoad(abs_path_file) self.jsonPath = currJsonPath tmpOutdict = copy.deepcopy(out_dict) for k1, v1 in tmpOutdict.items(): @@ -1372,7 +1372,7 @@ def __keyNameValidation(self, sInput): self.__reset() raise Exception(errorMsg) - def jsonLoad(self, jFile : str, masterFile : bool = True): + def jsonLoad(self, jFile : str): """ This method is the entry point of JsonPreprocessor. @@ -1386,12 +1386,6 @@ def jsonLoad(self, jFile : str, masterFile : bool = True): Path and name of main JSON file. The path can be absolute or relative and is also allowed to contain environment variables. -* ``masterFile`` - - / *Condition*: optional / *Type*: bool / - - Identifies the entry level when loading JSONP file in comparison with imported files levels. Default value is True - **Returns:** * ``oJson`` @@ -1400,6 +1394,8 @@ def jsonLoad(self, jFile : str, masterFile : bool = True): Preprocessed JSON file(s) as Python dictionary """ + # Identifies the entry level when loading JSONP file in comparison with imported files levels. + masterFile = True if self.recursive_level==0 else False jFile = CString.NormalizePath(jFile, sReferencePathAbs=os.path.dirname(os.path.abspath(sys.argv[0]))) self.handlingFile.append(jFile) if masterFile: @@ -1415,9 +1411,9 @@ def jsonLoad(self, jFile : str, masterFile : bool = True): except Exception as reason: self.__reset() raise Exception(f"Could not read json file '{jFile}' due to: '{reason}'!") - return self.jsonLoads(sJsonData, firstLevel=masterFile) + return self.jsonLoads(sJsonData) - def jsonLoads(self, sJsonpContent : str, referenceDir : str = '', firstLevel : bool = True): + def jsonLoads(self, sJsonpContent : str, referenceDir : str = ''): """ ``jsonLoads`` loads the JSONP content, preprocesses it and returns the preprocessed result as Python dictionary. @@ -1435,12 +1431,6 @@ def jsonLoads(self, sJsonpContent : str, referenceDir : str = '', firstLevel : b A reference path for loading imported files. -* ``firstLevel`` - - / *Condition*: optional / *Type*: bool / - - Identifies the entry level when loading JSONP content in comparison with imported files levels. - **Returns:** * ``oJson`` @@ -1517,6 +1507,8 @@ def __handleLastElement(sInput : str) -> str: sInput = sInput.replace(sParam, '"' + sParam + '"') return sInput + # Identifies the entry level when loading JSONP content in comparison with imported files levels. + firstLevel = True if self.recursive_level==0 else False if referenceDir != '': self.jsonPath = CString.NormalizePath(referenceDir, sReferencePathAbs=os.path.dirname(os.path.abspath(sys.argv[0]))) if not os.path.exists(self.jsonPath): From ffa5ed3b69e1480f766565404c6a53f870bb34f1 Mon Sep 17 00:00:00 2001 From: mas2hc Date: Tue, 27 Aug 2024 16:00:37 +0700 Subject: [PATCH 11/11] Update minor version release info tag --- config/robotframework_aio/release_items_JsonPreprocessor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/robotframework_aio/release_items_JsonPreprocessor.json b/config/robotframework_aio/release_items_JsonPreprocessor.json index 75ae0fea..4c99a686 100644 --- a/config/robotframework_aio/release_items_JsonPreprocessor.json +++ b/config/robotframework_aio/release_items_JsonPreprocessor.json @@ -105,7 +105,7 @@ " ] , - "0.12.0.;0.13.0." : [ + "0.12.1.;0.13.0." : [ " * Changed data type of return value