Skip to content

Improve overwrite parameter methodology #159

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Oct 10, 2023
Merged

Improve overwrite parameter methodology #159

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
06f51cd
Added self test cases JPP_0507 , JPP_0508, JPP_0509 based on latest i…
HolQue Oct 4, 2023
94242cc
Improve overwrite parameter methodology
namsonx Oct 5, 2023
7c11746
Merge pull request #158 from test-fullautomation/HolQue/task/selftest…
test-fullautomation Oct 5, 2023
dc266f9
Extended self test JPP_1000; added self test JPP_1001
HolQue Oct 5, 2023
e14b6f9
Merge pull request #160 from test-fullautomation/HolQue/task/selftest…
test-fullautomation Oct 5, 2023
995548f
Improve duplicated key handling for test scenario JPP_0507 and JPP_0509
namsonx Oct 9, 2023
25c2280
Tiny update a pattern to handle duplicated keys
namsonx Oct 9, 2023
7ab6c8e
Maintenance
HolQue Oct 9, 2023
dc3b7d1
Merge pull request #161 from test-fullautomation/HolQue/task/selftest…
test-fullautomation Oct 9, 2023
9a66c26
Fixed the issue related to testcase ID JPP_0358
namsonx Oct 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 107 additions & 16 deletions JsonPreprocessor/CJsonPreprocessor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,8 +250,26 @@ def __processImportFiles(self, input_data : dict) -> dict:
else:
tmpOutdict = copy.deepcopy(out_dict)
for k1, v1 in tmpOutdict.items():
pattern1 = "\${\s*[0-9A-Za-z_]+[0-9A-Za-z\.\-_]*\s*}(\[+\s*'.+'\s*\]+|\[+\s*\d+\s*\]+)*"
if re.search(pattern1, k1):
continue
pattern2 = "\${\s*[0-9A-Za-z\.\-_]*\.*" + k1 + "\s*}$|\[\s*'" + k1 + "'\s*\]$"
if re.search(pattern2, key):
key = k1
if k1 == key:
del out_dict[k1]
if isinstance(out_dict[key], list):
if out_dict[key][0] != "__handleDuplicatedKey__":
tmpValue = ["__handleDuplicatedKey__", out_dict[key], value]
del out_dict[key]
else:
tmpValue = out_dict[key]
tmpValue.append(value)
del out_dict[key]
else:
tmpValue = ["__handleDuplicatedKey__", out_dict[key], value]
del out_dict[key]
value = tmpValue
out_dict[key] = value
del tmpOutdict
out_dict[key] = value
return out_dict
Expand Down Expand Up @@ -537,13 +555,18 @@ def __jsonUpdated(k, v, oJson, bNested, keyNested = ''):
oJson[k] = v
globals().update({k:v})

def __loadNestedValue(sInputStr: str):
def __loadNestedValue(initValue: str, sInputStr: str):
bStringValue = False
if re.search("(str\(\s*" + pattern + "\))", sInputStr.lower()):
sInputStr = re.sub("str\(\s*(" + pattern + ")\s*\)", "$\\1", sInputStr)
bStringValue = True
else:
elif re.match("^\s*" + pattern + "\s*$", sInputStr):
sInputStr = re.sub("\$", "$$", sInputStr)
else:
while "str(" in initValue:
initValue = re.sub("str\((" + pattern + ")\)", "\\1", initValue, count=1)
raise Exception(f"Invalid syntax! An opened or closed curly brackets are missing in value '{initValue}'.\n \
Please check value '{initValue}' in config file!!!")
sInputStr = self.__checkParamName(sInputStr)
valueAfterProcessed = self.__nestedParamHandler(sInputStr)
for valueProcessed in valueAfterProcessed:
Expand Down Expand Up @@ -577,7 +600,7 @@ def __loadNestedValue(sInputStr: str):
keyNested = k
bNested = True
while "${" in k:
k = __loadNestedValue(k)
k = __loadNestedValue(keyNested, k)
elif re.match("^\s*" + pattern + "\s*$", k.lower()):
keyNested = k
k = re.sub("\$", "$$", k)
Expand All @@ -593,26 +616,42 @@ def __loadNestedValue(sInputStr: str):
v, bNested = self.__updateAndReplaceNestedParam(v, bNested, recursive=True)

elif isinstance(v, list):
tmpValue = []
for item in v:
if isinstance(item, str) and re.search(pattern, item.lower()):
bNested = True
while isinstance(item, str) and "${" in item:
item = __loadNestedValue(item)

tmpValue.append(item)
v = tmpValue
if v[0] != "__handleDuplicatedKey__":
tmpValue = []
for item in v:
if isinstance(item, str) and re.search(pattern, item.lower()):
bNested = True
initItem = item
while isinstance(item, str) and "${" in item:
item = __loadNestedValue(initItem, item)

tmpValue.append(item)
v = tmpValue
del tmpValue
else:
i=1
while i<len(v):
while re.search(pattern, v[i]):
bNested = True
initItem = v[i]
tmpValue = __loadNestedValue(initItem, v[i])
v[i] = tmpValue
tmpValue = v[i]
i+=1
v = tmpValue
del tmpValue

elif isinstance(v, str):
if re.search(pattern, v.lower()):
bNested = True
initValue = v
while isinstance(v, str) and "${" in v:
v = __loadNestedValue(v)
v = __loadNestedValue(initValue, v)

__jsonUpdated(k, v, oJson, bNested, keyNested)
if keyNested != '':
self.dUpdatedParams.update({k:v})

del tmpJson
return oJson, bNested

def __checkAndUpdateKeyValue(self, sInputStr: str) -> str:
Expand Down Expand Up @@ -781,6 +820,57 @@ def __handleListElements(sInput : str) -> str:
else:
newItem = newItem + self.__checkAndUpdateKeyValue(item)
return newItem

def __handleDuplicatedKey(dInput : dict) -> dict:
tmpDict = copy.deepcopy(dInput)
for k, v in tmpDict.items():
if isinstance(v, list) and v[0]=="__handleDuplicatedKey__":
tmpPattern = "\${\s*" + k + "\s*}|\${\s*" + k + "\.|\[\s*'"+ k + "'\s*\]|\." + k + "\.*"
if not re.search(pattern, str(v[-1])) or not re.search(tmpPattern, str(v[-1])):
dInput[k] = v[-1]
continue
else:
i=1
while i < len(v):
bRecursiveKey = False
if re.search(pattern, str(v[i])):
if isinstance(v[i], str):
if re.search(tmpPattern, v[i]):
v[i] = re.sub(k, k + "__RecursiveInitialValue__" + str(i-1), v[i])
bRecursiveKey = True
if isinstance(v[i], list):
newList = []
for item in v[i]:
if re.search(tmpPattern, item):
item = re.sub(k, k + "__RecursiveInitialValue__" + str(i-1), item)
bRecursiveKey = True
newList.append(item)
v[i] = newList
del newList
if bRecursiveKey:
k1 = k + "__RecursiveInitialValue__" + str(i)
dInput[k1] = v[i]
else:
k1 = k + "__RecursiveInitialValue__" + str(i)
dInput[k1] = v[i]
i+=1
dInput[k] = v[1] if len(v)==2 else v
if isinstance(v, dict):
dInput[k] = __handleDuplicatedKey(v)
del tmpDict
return dInput

def __removeDuplicatedKey(dInput : dict) -> dict:
if isinstance(dInput, dict):
for k, v in list(dInput.items()):
if "__RecursiveInitialValue__" in k:
del dInput[k]
else:
__removeDuplicatedKey(v)
elif isinstance(dInput, list):
for item in dInput:
__removeDuplicatedKey(item)


jFile = CString.NormalizePath(jFile, sReferencePathAbs=os.path.dirname(sys.argv[0]))
if not(os.path.isfile(jFile)):
Expand Down Expand Up @@ -878,6 +968,7 @@ def __handleListElements(sInput : str) -> str:
self.__checkDotInParamName(oJson)

if masterFile:
oJson = __handleDuplicatedKey(oJson)
for k, v in oJson.items():
if k in self.lDataTypes:
k = "JPavoidDataType_" + k
Expand Down Expand Up @@ -923,5 +1014,5 @@ def __handleListElements(sInput : str) -> str:
raise Exception(f"The variable '{parseNestedParam[0]}' is not available!")

self.__reset()

__removeDuplicatedKey(oJson)
return oJson
5 changes: 5 additions & 0 deletions test/JPP_TestUsecases.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ JPP_0201|PARAMETER_SUBSTITUTION|GOODCASE|JSON file with nested parameter / strin
JPP_0202|PARAMETER_SUBSTITUTION|GOODCASE|JSON file with nested parameter / index parameter substitution in parameter name / standard notation
JPP_0203|PARAMETER_SUBSTITUTION|GOODCASE|JSON file with nested parameter / index parameter substitution in parameter name / dotdict notation
JPP_0204|PARAMETER_SUBSTITUTION|GOODCASE|JSON file with nested parameter / index parameter substitution in parameter value / standard notation
JPP_0205|PARAMETER_SUBSTITUTION|GOODCASE|JSON file with nested parameter / index parameter substitution in parameter value / dotdict notation
JPP_0250|PARAMETER_SUBSTITUTION|BADCASE|JSON file with nested parameter / string parameter substitution in parameter value / innermost parameter not existing
JPP_0251|PARAMETER_SUBSTITUTION|BADCASE|JSON file with nested parameter / string parameter substitution in parameter name / in between parameter not existing
JPP_0252|PARAMETER_SUBSTITUTION|BADCASE|JSON file with nested parameter / index parameter substitution in parameter name / standard notation / index parameter not existing
Expand Down Expand Up @@ -41,6 +42,9 @@ JPP_0501|COMPOSITE_EXPRESSIONS|GOODCASE|JSON file with composite data structure
JPP_0502|COMPOSITE_EXPRESSIONS|GOODCASE|JSON file with composite data structure (nested lists and dictionaries 3 / some key names with dots inside)
JPP_0505|COMPOSITE_EXPRESSIONS|GOODCASE|JSON file with composite strings containing several times a colon and a comma (JSON syntax elements)
JPP_0506|COMPOSITE_EXPRESSIONS|GOODCASE|JSON file with composite strings containing several combinations of curly brackets and special characters before
JPP_0507|COMPOSITE_EXPRESSIONS|GOODCASE|JSON file containing several string concatenations in separate lines (1)
JPP_0508|COMPOSITE_EXPRESSIONS|GOODCASE|JSON file containing several string concatenations in separate lines (2)
JPP_0509|COMPOSITE_EXPRESSIONS|GOODCASE|JSON file containing several parameter assignments in separate lines (different syntax)
JPP_0550|COMPOSITE_EXPRESSIONS|BADCASE|JSON file with composite data structure (nested lists and dictionaries / some key names with dots inside)
JPP_0900|COMMON_SYNTAX_VIOLATIONS|GOODCASE|JSON file with syntax error, that is commented out
JPP_0950|COMMON_SYNTAX_VIOLATIONS|BADCASE|JSON file with syntax error (1)
Expand All @@ -49,6 +53,7 @@ JPP_0952|COMMON_SYNTAX_VIOLATIONS|BADCASE|JSON file with syntax error (3)
JPP_0953|COMMON_SYNTAX_VIOLATIONS|BADCASE|JSON file with syntax error (4): file is completely empty
JPP_0954|COMMON_SYNTAX_VIOLATIONS|BADCASE|JSON file with syntax error (5): file is empty (multiple pairs of brackets only)
JPP_1000|IMPLICIT_CREATION|GOODCASE|JSON file with dictionary keys to be created implicitly
JPP_1001|IMPLICIT_CREATION|GOODCASE|JSON file with dictionary keys to be created implicitly (same key names at all levels)
JPP_1150|CYCLIC_IMPORTS|BADCASE|JSON file with cyclic imports (JSON file imports itself)
JPP_1151|CYCLIC_IMPORTS|BADCASE|JSON file with cyclic imports (JSON file imports another file, that is already imported)
JPP_1200|PATH_FORMATS|GOODCASE|Relative path to JSON file
Loading