diff --git a/JsonPreprocessor/CJsonPreprocessor.py b/JsonPreprocessor/CJsonPreprocessor.py index 3ffc8ad7..d892ed35 100644 --- a/JsonPreprocessor/CJsonPreprocessor.py +++ b/JsonPreprocessor/CJsonPreprocessor.py @@ -57,11 +57,14 @@ class CSyntaxType(): class CPythonJSONDecoder(json.JSONDecoder): """ -**Method: PythonJSONDecoder** +**Class: PythonJSONDecoder** + Add python data types and syntax to json. ``True``, ``False`` and ``None`` will be a accepted as json syntax elements. **Args:** + **json.JSONDecoder** (*object*) + Decoder object provided by ``json.loads`` """ @@ -124,27 +127,27 @@ def custom_scan_once(self, string : str, idx : int) -> any: class CJsonPreprocessor(): """ -CJsonPreprocessor extends the syntax of json. +**Class: CJsonPreprocessor** -Features are + CJsonPreprocessor extends the syntax of json. -- Allow c/c++-style comments within json files. + Features are - // single line or part of single line and /\* \*/ multline comments are possible + - Allow c/c++-style comments within json files. -- Allow to import json files into json files + // single line or part of single line and /\* \*/ multline comments are possible - ``"[import]" : "relative/absolute path"``, imports another json file to exactly this location. + - Allow to import json files into json files - ``%envariable%`` and ``${envariable}`` can be used, too. + ``"[import]" : "relative/absolute path"``, imports another json file to exactly this location. -- Allow use of variables within json files + - Allow use of the defined paramaters within json files - In any place the syntax ``${basenode.subnode. ... nodename}`` allows to reference an already existing variable. + In any place the syntax ``${basenode.subnode. ... nodename}`` allows to reference an already existing parameter. - Example: + * Example: - .. code:: json + .. code:: { "basenode" : { @@ -154,23 +157,28 @@ class CJsonPreprocessor(): }, - "myVar" : "${basenode.subnode.myparam}" + "myVar" : ${basenode.subnode.myparam} } -- Allow python data types ``True``, ``False`` and ``None`` + - Allow Python data types ``True``, ``False`` and ``None`` """ def __init__(self, syntax: CSyntaxType = CSyntaxType.json , currentCfg : dict = {}) -> None: """ **Method: __init__** + Constructor **Args:** + **syntax** (*CSyntaxType*) optional + default: `json` , `python` + If set to `python`, then python data types are allowed as part of json file. **currentCfg** (*dict*) optional + Internally used to aggregate imported json files. """ self.lImportedFiles = [] @@ -184,26 +192,34 @@ def __init__(self, syntax: CSyntaxType = CSyntaxType.json , currentCfg : dict = def __sNormalizePath(self, sPath : str) -> str: """ **Method: __sNormalizePath** - Python struggles with - - UNC paths - e.g. ``\\hi-z4939\ccstg\....`` - - escape sequences in windows paths - e.g. ``c:\autotest\tuner \t`` will be interpreted as tab, the result + Python struggles with + + - UNC paths + + e.g. ``\\hi-z4939\ccstg\....`` + + - escape sequences in windows paths + + e.g. ``c:\autotest\tuner \t`` will be interpreted as tab, the result after processing it with an regexp would be ``c:\autotest uner`` - In order to solve this problems any slash will be replaced from backslash - to slash, only the two UNC backslashes must be kept if contained. + In order to solve this problems any slash will be replaced from backslash + to slash, only the two UNC backslashes must be kept if contained. **Args:** + **sPath** (*string*) + Absolute or relative path as input. Allows environment variables with ``%variable%`` or ``${variable}`` syntax. **Returns:** + **sPath** (*string*) - normalized path as string + + Normalized path as string """ # make all backslashes to slash, but mask # UNC indicator \\ before and restore after. @@ -240,19 +256,22 @@ def __mkslash(sPath : str) -> str: def __processImportFiles(self, input_data : dict) -> dict: ''' **Method: __processImportFiles** - this is a custom decorder of ``json.loads object_pairs_hook`` function. + + This is a custom decorder of ``json.loads object_pairs_hook`` function. This method helps to import json files which are provided in ``"[import]"`` keyword into the current json file. **Args:** + **input_data** (*dict*) - dictionary from json file as input + + Dictionary from json file as input **Returns:** + **out_dict** (*dict*) - dictionary as output - dictionary is extended if ``"[import]"`` found and properly imported. + Dictionary as output, dictionary is extended if ``"[import]"`` found and properly imported. ''' out_dict = {} @@ -279,18 +298,23 @@ def __processImportFiles(self, input_data : dict) -> dict: def __load_and_removeComments(self, jsonFile : str) -> str: """ **Method: __load_and_removeComments** - loads a given json file and filters all C/C++ style comments. + + Loads a given json file and filters all C/C++ style comments. **Args:** + **jsonFile** (*string*) - path (absolute/relative/) file to be processed. + + Path (absolute/relative/) file to be processed. The path can contain windows/linux style environment variables. !ATTENTION! This is dangerous **Returns:** + **sContentCleaned** (*string*) - string version of json file after removing all comments. + + String version of json file after removing all comments. """ def replacer(match): @@ -313,15 +337,20 @@ def replacer(match): def __nestedParamHandler(self, sInputStr : str) -> str: ''' **Method: __nestedParamHandler** - This method handles nested variables in parameter names or values. Variable syntax is ${Variable_Name}. + + This method handles nested variables in parameter names or values. Variable syntax is ${Variable_Name}. **Args:** - **sInputStr** (*string*) - Parameter name or value which contains a nested variable. + + **sInputStr** (*string*) + + Parameter name or value which contains a nested variable. **Returns:** - **sStrHandled** (*string*) - String which contains the resolved variable. + + **sStrHandled** (*string*) + + String which contains the resolved variable. ''' #globals().update(currentCfg) @@ -384,15 +413,20 @@ def __nestedParamHandler(self, sInputStr : str) -> str: def __updateAndReplaceNestedParam(self, oJson : dict, recursive : bool = False): ''' **Method: __updateAndReplaceNestedParam** + This method replaces all nested parameters in key and value of a json object . **Args:** + **oJson** (*dict*) + Input Json object as dictionary. This dictionary will be searched for all ``${variable}`` occurences. If found it will be replaced with it's current value. **Returns:** + **oJsonOut** (*dict*) + Output Json object as dictionary with all variables resolved. ''' @@ -472,19 +506,24 @@ def __updateAndReplaceNestedParam(self, oJson : dict, recursive : bool = False): def jsonLoad(self, jFile : str, masterFile : bool = True): ''' **Method: jsonLoad** + This function is the entry point of JsonPreprocessor. It loads the json file, preprocesses it and returns the preprocessed result as data structure. **Args:** + **jFile** (*string*) - relative/absolute path to main json file. + + Relative/absolute path to main json file. ``%envvariable%`` and ``${envvariable}`` can be used, too in order to access environment variables. -**Returns:** +**Returns:** + **oJson** (*dict*) - preprocessed json file(s) as data structure + + Preprocessed json file(s) as dictionary data structure ''' jFile=jFile.strip() diff --git a/JsonPreprocessor/JsonPreprocessor.pdf b/JsonPreprocessor/JsonPreprocessor.pdf index 5de91251..fc4ab6f4 100644 Binary files a/JsonPreprocessor/JsonPreprocessor.pdf and b/JsonPreprocessor/JsonPreprocessor.pdf differ diff --git a/README.md b/README.md index 63f1711e..e3fbbd92 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ After the build processes is completed, the package is located in A detailed documentation of the Json Preprocessor\'s package can be found here: -[Json-Preprocessor.pdf](https://github.com/test-fullautomation/python-jsonpreprocessor/blob/develop/doc/_build/latex/Json-Preprocessor.pdf) +[JsonPreprocessor.pdf](https://github.com/test-fullautomation/python-jsonpreprocessor/blob/develop/JsonPreprocessor/JsonPreprocessor.pdf) ## Feedback diff --git a/packagedoc/additional_docs/Description.rst b/packagedoc/additional_docs/Description.rst index 66b4088c..c61474a5 100644 --- a/packagedoc/additional_docs/Description.rst +++ b/packagedoc/additional_docs/Description.rst @@ -12,13 +12,401 @@ See the License for the specific language governing permissions and limitations under the License. -The json preprocessor: +How to install +-------------- -.. image:: ./pictures/python3-jsonpreprocessor.png +Firstly, clone `python-jsonpreprocessor `_ +repository to your machine. +Then open the folder in which you have cloned the repository python-jsonpreprocessor, following the commands +below to build or install this package: +.. code:: -TO BE CONTINUED + setup.py build will build the package underneath 'build/' + setup.py install will install the package +After the build processes is completed, the package is located in 'build/', and the generated package +documentation is located in **/JsonPreprocessor**. +We can use ``--help`` to discover the options for ``build`` command, example: + +.. code:: + + setup.py build will build the package underneath 'build/' + setup.py install will install the package + +Features +-------- + +Basic Json format +~~~~~~~~~~~~~~~~~ + +Users can use JsonPreprocessor to handle the json file with its original format. + +**Example:** + +.. code:: + + { + "Project": "name_of_prject", + "version": { + "major": "0", + "minor": "1", + "patch": "1" + }, + "params": { + "global": { + "param_1" : "value_1", + "param_2" : value_2, + "structure_param": { + "general": "general" + } + } + }, + "device" : "device_name" + } + +Adding the comments +~~~~~~~~~~~~~~~~~~~ + +Often large projects require a lot of configuration parameters. So adding comments to json files is +useful in case of more and more content is added, e.g. because of a json file has to hold a huge number +of configuration parameters for different features. Comments can be used here to clarify the meaning of +these parameters or the differences between them. + +Every line starting with **"//"**, is commented out. Therefore a comment is valid for singles lines only. + +Comment out a block of several lines with only one start and one end comment string, is currently not supported. + +**Example:** + +.. code:: + + //***************************************************************************** + // Author: ROBFW-AIO Team + // + // This file defines all common global parameters and will be included to all + // test config files + //***************************************************************************** + { + "Project": "name_of_prject", + // + "version": { + "majorversion": "0", + "minorversion": "1", + "patchversion": "1" + }, + "params": { + // + "global": { + "param_1" : "value_1", + "param_2" : value_2, // + "structure_param": { + "general": "general" + } + } + }, + "device" : "device_name" + } + +Imports other json files +~~~~~~~~~~~~~~~~~~~~~~~~ + +This import feature enables developers to take over the content of other json files into the +current json file. A json file that is imported into another json file, can contain imports also +(allows nested imports). + +A possible usecase for nested imports is to handle configuration parameters of different variants +of a feature or a component within a bunch of several smaller files, instead of putting all parameter +into only one large json file. + +**Example:** + +Suppose we have the json file ``params_global.json`` with the content: + +.. code:: + + //***************************************************************************** + // Author: ROBFW-AIO Team + // + // This file defines all common global parameters and will be included to all + // test config files + //***************************************************************************** + // + // This is to distinguish the different types of resets + { + "import_param_1" : "value_1", + + "import_param_2" : "value_2", + + "import_structure_1": { // + "general": "general" + } + } + +And other json file ``preprocessor_definitions.json`` with content: + +.. code:: + + //***************************************************************************** + // Author: ROBFW-AIO Team + // + // This file defines all common global parameters and will be included to all + // test config files + //***************************************************************************** + { + "import_param_3" : "value_3", + + "import_param_4" : "value_4", + + // + + "import_structure_2": { + "general": "general" + } + } + +Then we can import these 2 files above to the json file ``config.json`` with the [import] statement: + +.. code:: + + //***************************************************************************** + // Author: ROBFW-AIO Team + // + // This file defines all common global parameters and will be included to all + // test config files + //***************************************************************************** + { + "Project": "name_of_prject", + "version": { + "major": "0", + "minor": "1", + "patch": "1" + }, + "params": { + "global": { + "[import]": "/params_global.json" + } + }, + "preprocessor": { + "definitions": { + "[import]": "/preprocessor_definitions.json" + } + }, + "device" : "device_name" + } + +After all imports are resolved by the JsonPreprocessor, this is the resulting of data structure: + +.. code:: + + { + "Project": "name_of_prject", + "version": { + "major": "0", + "minor": "1", + "patch": "1" + }, + "params": { + "global": { + "import_param_1" : "value_1", + "import_param_2" : "value_2", + "import_structure_1": { + "general": "general" + } + } + }, + "preprocessor": { + "definitions": { + "import_param_3" : "value_3", + "import_param_4" : "value_4", + "import_structure_2": { + "general": "general" + } + } + }, + "device" : "device_name" + } + + Add new or overwrites existing parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This JsonPreprocessor package also provides developers ability to add new as well as overwrite +existing parameters. Developers can update parameters which are already declared and add new +parameters or new element into existing parameters. The below example will show the way to do +these features. + +In case we have many different variants, and each variant requires a different value assigned +to the parameter, users can use this feature to add new parameters and update new values for +existing parameters of existing configuation object. + +**Example:** + +Suppose we have the json file ``params_global.json`` with the content: + +.. code:: + + //***************************************************************************** + // Author: ROBFW-AIO Team + // + // This file defines all common global parameters and will be included to all + // test config files + //***************************************************************************** + // + // This is to distinguish the different types of resets + { + "import_param_1" : "value_1", + + "import_param_2" : "value_2", + + "import_structure_1": { // + "general": "general" + } + } + +Then we import ``params_global.json`` to json file ``config.json`` with content: + +.. code:: + + { + "Project": "name_of_prject", + "version": { + "major": "0", + "minor": "1", + "patch": "1" + }, + "params": { + "global": { + "[import]": "/params_global.json" + } + }, + "device" : "device_name", + // Overwrite parameters + "${params}['global']['import_param_1']": "new_value_1", + "${version}['patch']": "2", + // Add new parameters + "new_param": { + "abc": 9, + "xyz": "new param" + }, + "${params}['global']['import_structure_1']['new_structure_param']": "new_structure_value" + } + +After all imports are resolved by the JsonPreprocessor, this is the resulting of data structure: + +.. code:: + + { + "Project": "name_of_prject", + "version": { + "major": "0", + "minor": "1", + "patch": "2" + }, + "params": { + "global": { + "import_param_1" : "new_value_1", + "import_param_2" : "value_2", + "import_structure_1": { + "general": "general", + "new_structure_param": "new_structure_value" + } + } + }, + "device" : "device_name", + "new_param": { + "abc": 9, + "xyz": "new param" + } + } + +Using defined parameters +~~~~~~~~~~~~~~~~~~~~~~~~ + +With JsonPreprocessor package, users can also use the defined parameters in Json file. The value of +the defined parameter could be called with syntax ``${}`` + +**Example:** + +Suppose we have the json file ``config.json`` with the content: + +.. code:: + + { + "Project": "name_of_prject", + "version": { + "major": "0", + "minor": "1", + "patch": "1" + }, + "params": { + "global": { + "import_param_1" : "value_1", + "import_param_2" : "value_2", + "import_structure_1": { + "general": "general" + } + } + }, + "preprocessor": { + "definitions": { + "import_param_3" : "value_3", + "import_param_4" : "value_4", + "ABC": "param_ABC", + "import_structure_1": { + "general": "general" + } + } + }, + "device" : "device_name", + // Using the defined parameters + "${params}['global'][${preprocessor}['definitions']['ABC']]": True, + "${params}['global']['import_param_1']": ${preprocessor}['definitions']['import_param_4'] + } + +After all imports are resolved by the JsonPreprocessor, this is the resulting of data structure: + +.. code:: + + { + "Project": "name_of_prject", + "version": { + "major": "0", + "minor": "1", + "patch": "1" + }, + "params": { + "global": { + "import_param_1" : "value_4", + "import_param_2" : "value_2", + "import_structure_1": { + "general": "general" + }, + "param_ABC": True + } + }, + "preprocessor": { + "definitions": { + "import_param_3" : "value_3", + "import_param_4" : "value_4", + "ABC": "param_ABC", + "import_structure_1": { + "general": "general" + } + } + }, + "TargetName" : "device_name" + } + +Accepted ``True``, ``False``, and ``None`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some keywords are different between Json and Python syntax: + +* Json syntax: **``true``**, **``false``**, **``null``** + +* Python syntax: **``True``**, **``False``**, **``None``** + +To facilitate the usage of configuration files in Json format, both ways of syntax are accepted. diff --git a/packagedoc/additional_docs/History.tex b/packagedoc/additional_docs/History.tex index d529af95..d5ea9f62 100644 --- a/packagedoc/additional_docs/History.tex +++ b/packagedoc/additional_docs/History.tex @@ -16,5 +16,7 @@ \begin{tabular}{| m{6em} | m{26em} |}\hline \textbf{0.1.0} & 01/2022\\ \hline \multicolumn{2}{| m{32em} |}{\textit{Initial version}}\\ \hline + \textbf{0.1.4} & 09/2022\\ \hline + \multicolumn{2}{| m{32em} |}{\textit{Updated documentation}}\\ \hline \end{tabular} \end{center} diff --git a/packagedoc/additional_docs/Introduction.rst b/packagedoc/additional_docs/Introduction.rst index 7e72e6c3..1005b2ac 100644 --- a/packagedoc/additional_docs/Introduction.rst +++ b/packagedoc/additional_docs/Introduction.rst @@ -12,6 +12,26 @@ See the License for the specific language governing permissions and limitations under the License. -*Json Preprocessor* +Json Preprocessor documentation +=============================== -!!! add introduction - if wanted !! +**This is the documentation for Python JsonPreprocessor** + +Json is a format used to represent data and becomes the universal standard of data +exchange. Today many software projects are using configuration file in Json format. +For a big or a complex project there is a need to have some enhanced format in Json +file such as adding the comments, importing other Json files, etc. + +Based on that needs, we develop JsonPreprocessor package: + +* Gives the possibility to comment out parts of the content. This feature can be used to + explain the meaning of the parameters defined inside the configuration files. + +* Has ability to import other Json files. This feature can be applied for complex project, + users can create separated Json files then importing them to other Json file. + +* Allows users using the defined parameter in Json file. + +* Accepts **``True``**, **``False``**, and **``None``** in Json syntax + + .. image:: ./pictures/python3-jsonpreprocessor.png