diff --git a/.gitignore b/.gitignore index dccbe166..e70c1b97 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ target .pydevproject +# VSCode IDE +.vscode + # Packages *.egg *.egg-info diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9a48f3ad..65bdba0e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,7 @@ v2.2.2 *Release date: In development* - Add missing param in download_videos method to fix error downloading videos from a remote server +- Add map_param function to dataset module - New param [RANDOM_PHONE_NUMBER] in *replace_param* method to generate random phone number v2.2.1 diff --git a/docs/bdd_integration.rst b/docs/bdd_integration.rst index 30ed4f24..4bfadac7 100644 --- a/docs/bdd_integration.rst +++ b/docs/bdd_integration.rst @@ -144,6 +144,54 @@ When this happens, steps of the affected scenarios for that precondition are not first step defined in those scenarios will be automatically failed because of that precondition exception, in order to properly fail the execution and show the stats. +Behave variables transformation +------------------------------- + +Toolium provides a set of functions that allow the transformation of specific string tags into different values. +These are the main ones, along with the list of tags they support and their associated replacement logic (click on the +functions or check the :ref:`dataset ` module for more implementation details): + +`replace_param `_: + +* :code:`[STRING_WITH_LENGTH_XX]`: Generates a fixed length string +* :code:`[INTEGER_WITH_LENGTH_XX]`: Generates a fixed length integer +* :code:`[STRING_ARRAY_WITH_LENGTH_XX]`: Generates a fixed length array of strings +* :code:`[INTEGER_ARRAY_WITH_LENGTH_XX]`: Generates a fixed length array of integers +* :code:`[JSON_WITH_LENGTH_XX]`: Generates a fixed length JSON +* :code:`[MISSING_PARAM]`: Generates a None object +* :code:`[NULL]`: Generates a None object +* :code:`[TRUE]`: Generates a boolean True +* :code:`[FALSE]`: Generates a boolean False +* :code:`[EMPTY]`: Generates an empty string +* :code:`[B]`: Generates a blank space +* :code:`[RANDOM]`: Generates a random value +* :code:`[RANDOM_PHONE_NUMBER]`: Generates a random phone number following the pattern +34654XXXXXX +* :code:`[TIMESTAMP]`: Generates a timestamp from the current time +* :code:`[DATETIME]`: Generates a datetime from the current time +* :code:`[NOW]`: Similar to DATETIME without milliseconds; the format depends on the language +* :code:`[NOW + 2 DAYS]`: Similar to NOW but two days later +* :code:`[NOW - 1 MINUTES]`: Similar to NOW but one minute earlier +* :code:`[TODAY]`: Similar to NOW without time; the format depends on the language +* :code:`[TODAY + 2 DAYS]`: Similar to NOW, but two days later +* :code:`[STR:xxxx]`: Cast xxxx to a string +* :code:`[INT:xxxx]`: Cast xxxx to an int +* :code:`[FLOAT:xxxx]`: Cast xxxx to a float +* :code:`[LIST:xxxx]`: Cast xxxx to a list +* :code:`[DICT:xxxx]`: Cast xxxx to a dict +* :code:`[UPPER:xxxx]`: Converts xxxx to upper case +* :code:`[LOWER:xxxx]`: Converts xxxx to lower case + +`map_param `_: + +* :code:`[CONF:xxxx]`: Value from the config dict in context.project_config for the key xxxx +* :code:`[LANG:xxxx]`: String from the texts dict in context.language_dict for the key xxxx, using the language specified in context.language +* :code:`[POE:xxxx]`: Definition(s) from the POEditor terms list in context.poeditor_terms for the term xxxx (see :ref:`poeditor ` module for details) +* :code:`[TOOLIUM:xxxx]`: Value from the toolium config in context.toolium_config for the key xxxx +* :code:`[CONTEXT:xxxx]`: Value from the context storage dict for the key xxxx, or value of the context attribute xxxx, if the former does not exist +* :code:`[ENV:xxxx]`: Value of the OS environment variable xxxx +* :code:`[FILE:xxxx]`: String with the content of the file in the path xxxx +* :code:`[BASE64:xxxx]`: String with the base64 representation of the file content in the path xxxx + Lettuce ~~~~~~~ diff --git a/docs/toolium.rst b/docs/toolium.rst index 3263b326..c0205834 100644 --- a/docs/toolium.rst +++ b/docs/toolium.rst @@ -72,6 +72,26 @@ jira :undoc-members: :show-inheritance: +.. _pytest_fixtures: + +pytest_fixtures +--------------- + +.. automodule:: toolium.pytest_fixtures + :members: + :undoc-members: + :show-inheritance: + +.. _selenoid: + +selenoid +-------- + +.. automodule:: toolium.selenoid + :members: + :undoc-members: + :show-inheritance: + .. _test_cases: test_cases diff --git a/docs/toolium.utils.rst b/docs/toolium.utils.rst index b588d193..09642e84 100644 --- a/docs/toolium.utils.rst +++ b/docs/toolium.utils.rst @@ -1,7 +1,29 @@ +.. _utils: + utils ===== -.. _utils: +.. _dataset: + +dataset +------- + +.. automodule:: toolium.utils.dataset + :members: + :undoc-members: + :show-inheritance: + +.. _download_files: + +download_files +-------------- + +.. automodule:: toolium.utils.download_files + :members: + :undoc-members: + :show-inheritance: + +.. _driver_utils: driver_utils ------------ @@ -11,6 +33,18 @@ driver_utils :undoc-members: :show-inheritance: +.. _driver_wait_utils: + +driver_wait_utils +----------------- + +.. automodule:: toolium.utils.driver_utils + :members: + :undoc-members: + :show-inheritance: + +.. _path_utils: + path_utils ---------- @@ -18,3 +52,13 @@ path_utils :members: :undoc-members: :show-inheritance: + +.. _poeditor: + +poeditor +-------- + +.. automodule:: toolium.utils.poeditor + :members: + :undoc-members: + :show-inheritance: diff --git a/toolium/test/resources/document.txt b/toolium/test/resources/document.txt new file mode 100644 index 00000000..2d60b2e4 --- /dev/null +++ b/toolium/test/resources/document.txt @@ -0,0 +1 @@ +Document used to verify functionalities in MSS \ No newline at end of file diff --git a/toolium/test/resources/toolium.cfg b/toolium/test/resources/toolium.cfg new file mode 100644 index 00000000..6df13625 --- /dev/null +++ b/toolium/test/resources/toolium.cfg @@ -0,0 +1,7 @@ +[Driver] +# Valid driver types: firefox, chrome, iexplore, edge, safari, opera, phantomjs, ios, android +type: firefox + +[TestExecution] +environment: QA +language: EN diff --git a/toolium/test/utils/test_dataset_map_param.py b/toolium/test/utils/test_dataset_map_param.py new file mode 100644 index 00000000..ca168d81 --- /dev/null +++ b/toolium/test/utils/test_dataset_map_param.py @@ -0,0 +1,392 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2022 Telefónica Investigación y Desarrollo, S.A.U. +This file is part of Toolium. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import mock +import os +import pytest + +from toolium.config_parser import ExtendedConfigParser +from toolium.utils.dataset import map_param +from toolium.utils.dataset import hide_passwords + + +def test_a_env_param(): + """ + Verification of a mapped parameter as ENV + """ + os.environ['MY_PASSWD'] = "admin123" + result = map_param("[ENV:MY_PASSWD]") + expected = "admin123" + assert expected == result + + +def test_a_file_param(): + """ + Verification of a mapped parameter as FILE + """ + result = map_param("[FILE:toolium/test/resources/document.txt]") + expected = "Document used to verify functionalities in MSS " + assert expected == result + + +def test_a_base64_param(): + """ + Verification of a mapped parameter as BASE64 + """ + result = map_param("[BASE64:toolium/test/resources/document.txt]") + expected = "RG9jdW1lbnQgdXNlZCB0byB2ZXJpZnkgZnVuY3Rpb25hbGl0aWVzIGluIE1TUyA=" + assert expected == result + + +def test_a_lang_param(): + """ + Verification of a mapped parameter as LANG + """ + context = mock.MagicMock() + context.language_dict = {"home": {"button": {"send": {"es": "enviar", "en": "send"}}}} + context.language = "es" + result = map_param("[LANG:home.button.send]", context) + expected = "enviar" + assert expected == result + + +def test_a_toolium_param(): + """ + Verification of a mapped parameter as TOOLIUM + """ + context = mock.MagicMock() + config_file_path = os.path.join("toolium", "test", "resources", "toolium.cfg") + context.toolium_config = ExtendedConfigParser.get_config_from_file(config_file_path) + result = map_param("[TOOLIUM:TestExecution_environment]", context) + expected = "QA" + assert expected == result + + +def test_a_conf_param(): + """ + Verification of a mapped parameter as CONF + """ + context = mock.MagicMock() + context.project_config = {"service": {"port": 80}} + result = map_param("[CONF:service.port]", context) + expected = 80 + assert expected == result + + +def test_a_context_param(): + """ + Verification of a mapped parameter as CONTEXT + """ + context = mock.MagicMock() + context.attribute = "attribute value" + context.storage = {"storage_key": "storage entry value"} + result_att = map_param("[CONTEXT:attribute]", context) + expected_att = "attribute value" + assert expected_att == result_att + result_st = map_param("[CONTEXT:storage_key]", context) + expected_st = "storage entry value" + assert expected_st == result_st + + +def test_a_poe_param_single_result(): + """ + Verification of a POE mapped parameter with a single result for a reference + """ + context = mock.MagicMock() + context.poeditor_export = [ + { + "term": "Poniendo mute", + "definition": "Ahora la tele está silenciada", + "reference": "home:home.tv.mute", + } + ] + result = map_param('[POE:home.tv.mute]', context) + expected = "Ahora la tele está silenciada" + assert result == expected + + +def test_a_poe_param_no_result_assertion(): + """ + Verification of a POE mapped parameter without result + """ + context = mock.MagicMock() + context.poeditor_export = [ + { + "term": "Poniendo mute", + "definition": "Ahora la tele está silenciada", + "reference": "home:home.tv.mute", + } + ] + with pytest.raises(Exception) as excinfo: + map_param('[POE:home.tv.off]', context) + assert "No translations found in POEditor for reference home.tv.off" in str(excinfo.value) + + +def test_a_poe_param_prefix_with_no_definition(): + """ + Verification of a POE mapped parameter with a single result for a reference + """ + context = mock.MagicMock() + context.project_config = {'poeditor': {'key_field': 'reference', 'search_type': 'contains', 'prefixes': ['PRE.']}} + context.poeditor_export = [ + { + "term": "Hola, estoy aquí para ayudarte", + "definition": None, + "reference": "common:PRE.common.greetings.main", + }, + { + "term": "Hola! En qué puedo ayudarte?", + "definition": "Hola, buenas", + "reference": "common:common.greetings.main", + } + ] + result = map_param('[POE:common:common.greetings.main]', context) + expected = "Hola, buenas" + assert result == expected + + +def test_a_poe_param_single_result_selecting_a_key_field(): + """ + Verification of a POE mapped parameter with a single result for a term + """ + context = mock.MagicMock() + context.project_config = {'poeditor': {'key_field': 'term'}} + context.poeditor_export = [ + { + "term": "loginSelectLine_text_subtitle", + "definition": "Te damos la bienvenida", + "context": "", + "term_plural": "", + "reference": "", + "comment": "" + } + ] + result = map_param('[POE:loginSelectLine_text_subtitle]', context) + expected = "Te damos la bienvenida" + assert result == expected + + +def test_a_poe_param_multiple_results(): + """ + Verification of a POE mapped parameter with several results for a reference + """ + context = mock.MagicMock() + context.project_config = {'poeditor': {'key_field': 'reference'}} + context.poeditor_export = [ + { + "term": "Hola, estoy aquí para ayudarte", + "definition": "Hola, estoy aquí para ayudarte", + "reference": "common:common.greetings.main", + }, + { + "term": "Hola! En qué puedo ayudarte?", + "definition": "Hola, buenas", + "reference": "common:common.greetings.main", + } + ] + result = map_param('[POE:common.greetings.main]', context) + first_expected = "Hola, estoy aquí para ayudarte" + second_expected = "Hola, buenas" + assert len(result) == 2 + assert result[0] == first_expected + assert result[1] == second_expected + + +def test_a_poe_param_multiple_options_but_only_one_result(): + """ + Verification of a POE mapped parameter with a single result from several options for a key + """ + context = mock.MagicMock() + context.project_config = {'poeditor': {'key_field': 'term', 'search_type': 'exact'}} + context.poeditor_export = [ + { + "term": "loginSelectLine_text_subtitle", + "definition": "Te damos la bienvenida_1", + "context": "", + "term_plural": "", + "reference": "", + "comment": "" + }, + { + "term": "loginSelectLine_text_subtitle_2", + "definition": "Te damos la bienvenida_2", + "context": "", + "term_plural": "", + "reference": "", + "comment": "" + } + ] + result = map_param('[POE:loginSelectLine_text_subtitle]', context) + expected = "Te damos la bienvenida_1" + assert result == expected + + +def test_a_poe_param_with_prefix(): + """ + Verification of a POE mapped parameter with several results for a reference, filtered with a prefix + """ + context = mock.MagicMock() + context.project_config = {'poeditor': {'key_field': 'reference', 'search_type': 'contains', 'prefixes': ['PRE.']}} + context.poeditor_export = [ + { + "term": "Hola, estoy aquí para ayudarte", + "definition": "Hola, estoy aquí para ayudarte", + "reference": "common:common.greetings.main", + }, + { + "term": "Hola! En qué puedo ayudarte?", + "definition": "Hola, buenas", + "reference": "common:PRE.common.greetings.main", + } + ] + result = map_param('[POE:common.greetings.main]', context) + expected = "Hola, buenas" + assert result == expected + + +def test_a_poe_param_with_two_prefixes(): + """ + Verification of a POE mapped parameter with several results for a reference, filtered with two prefixes + """ + context = mock.MagicMock() + context.project_config = {'poeditor': {'prefixes': ['MH.', 'PRE.']}} + context.poeditor_export = [ + { + "term": "Hola, estoy aquí para ayudarte", + "definition": "Hola, estoy aquí para ayudarte", + "reference": "common:common.greetings.main", + }, + { + "term": "Hola! En qué puedo ayudarte?", + "definition": "Hola, buenas", + "reference": "common:PRE.common.greetings.main", + }, + { + "term": "Hola! En qué puedo ayudarte MH?", + "definition": "Hola, buenas MH", + "reference": "common:MH.common.greetings.main", + } + ] + result = map_param('[POE:common.greetings.main]', context) + expected = "Hola, buenas MH" + assert result == expected + + +def test_a_poe_param_with_prefix_and_exact_resource(): + """ + Verification of a POE mapped parameter that uses an exact resource name and has a prefix configured + """ + context = mock.MagicMock() + context.project_config = {'poeditor': {'prefixes': ['PRE.']}} + context.poeditor_export = [ + { + "term": "Hola, estoy aquí para ayudarte", + "definition": "Hola, estoy aquí para ayudarte", + "reference": "common:common.greetings.main", + }, + { + "term": "Hola! En qué puedo ayudarte?", + "definition": "Hola, buenas", + "reference": "common:PRE.common.greetings.main", + } + ] + result = map_param('[POE:common:common.greetings.main]', context) + expected = "Hola, buenas" + assert result == expected + + +def test_a_text_param(): + """ + Verification of a text param + """ + result = map_param("just_text") + expected = "just_text" + assert expected == result + + +def test_a_combi_of_textplusconfig(): + """ + Verification of a combination of text plus a config param + """ + os.environ['MY_VAR'] = "some value" + result = map_param("adding [ENV:MY_VAR]") + expected = "adding some value" + assert expected == result + + +def test_a_combi_of_textplusconfig_integer(): + """ + Verification of a combination of text plus a config param + """ + context = mock.MagicMock() + context.project_config = {"service": {"port": 80}} + result = map_param("use port [CONF:service.port]", context) + expected = "use port 80" + assert expected == result + + +def test_a_combi_of_configplusconfig(): + """ + Verification of a combination of a config param plus a config param + """ + os.environ['MY_VAR_1'] = "this is " + os.environ['MY_VAR_2'] = "some value" + result = map_param("[ENV:MY_VAR_1][ENV:MY_VAR_2]") + expected = "this is some value" + assert expected == result + + +def test_a_combi_of_config_plustext_plusconfig(): + """ + Verification of a combination of a config param plus text plus a config param + """ + os.environ['MY_VAR_1'] = "this is" + os.environ['MY_VAR_2'] = "some value" + result = map_param("[ENV:MY_VAR_1] some text and [ENV:MY_VAR_2]") + expected = "this is some text and some value" + assert expected == result + + +def test_a_conf_param_with_special_characters(): + """ + Verification of a combination of text plus a config param with special characters + """ + context = mock.MagicMock() + context.project_config = {"user": "user-1", "password": "p4:ssw0_rd"} + result = map_param("[CONF:user]-an:d_![CONF:password]", context) + expected = "user-1-an:d_!p4:ssw0_rd" + assert expected == result + + +# Data input for hide_passwords +hide_passwords_data = [ + ('name', 'value', 'value'), + ('key', 'value', '*****'), + ('second_key', 'value', '*****'), + ('pas', 'value', 'value'), + ('pass', 'value', '*****'), + ('password', 'value', '*****'), + ('secret', 'value', '*****'), + ('code', 'value', '*****'), + ('token', 'value', '*****'), +] + + +@pytest.mark.parametrize("key,value,hidden_value", hide_passwords_data) +def test_hide_passwords(key, value, hidden_value): + assert hidden_value == hide_passwords(key, value) diff --git a/toolium/test/utils/test_dataset_utils.py b/toolium/test/utils/test_dataset_replace_param.py similarity index 100% rename from toolium/test/utils/test_dataset_utils.py rename to toolium/test/utils/test_dataset_replace_param.py diff --git a/toolium/test/utils/test_poeditor.py b/toolium/test/utils/test_poeditor.py new file mode 100644 index 00000000..333025ef --- /dev/null +++ b/toolium/test/utils/test_poeditor.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2022 Telefónica Investigación y Desarrollo, S.A.U. +This file is part of Toolium. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import mock + +from toolium.utils.poeditor import get_valid_lang + + +def test_poe_lang_param(): + """ + Verification of a POEditor language param + """ + context = mock.MagicMock() + context.poeditor_language_list = ['en-gb', 'de', 'pt-br', 'es', 'es-ar', 'es-cl', 'es-co', 'es-ec'] + assert get_valid_lang(context, 'pt-br') == 'pt-br' + assert get_valid_lang(context, 'es') == 'es' + assert get_valid_lang(context, 'es-es') == 'es' + assert get_valid_lang(context, 'es-co') == 'es-co' diff --git a/toolium/utils/dataset.py b/toolium/utils/dataset.py index 0c15a01b..4405aef9 100644 --- a/toolium/utils/dataset.py +++ b/toolium/utils/dataset.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Copyright 2021 Telefónica Investigación y Desarrollo, S.A.U. +Copyright 2022 Telefónica Investigación y Desarrollo, S.A.U. This file is part of Toolium. Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,13 +16,16 @@ limitations under the License. """ +import os import re import datetime import logging import random as r import string import json +import base64 from ast import literal_eval +from copy import deepcopy logger = logging.getLogger(__name__) @@ -62,6 +65,7 @@ def replace_param(param, language='es', infer_param_type=True): this function also tries to infer and cast the result to the most appropriate data type, attempting first the direct conversion to a Python built-in data type and then, if not possible, the conversion to a dict/list parsing the string as a JSON object/list. + :param param: parameter value :param language: language to configure date format for NOW and TODAY ('es' or other) :param infer_param_type: whether to infer and change the data type of the result or not @@ -90,14 +94,16 @@ def replace_param(param, language='es', infer_param_type=True): if param != new_param: if type(new_param) == str: - logger.debug('Replaced param from "%s" to "%s"' % (param, new_param)) + logger.debug(f'Replaced param from "{param}" to "{new_param}"') else: - logger.debug('Replaced param from "%s" to %s' % (param, new_param)) + logger.debug(f'Replaced param from "{param}" to {new_param}') return new_param def _replace_param_type(param): - """Replace param to a new param type + """ + Replace param with a new param type. + Available replacements: [MISSING_PARAM], [TRUE], [FALSE], [NULL] :param param: parameter value :return: tuple with replaced value and boolean to know if replacement has been done @@ -119,7 +125,8 @@ def _replace_param_type(param): def _replace_param_replacement(param, language): - """Replace partial param value. + """ + Replace param with a new param value. Available replacements: [EMPTY], [B], [RANDOM], [TIMESTAMP], [DATETIME], [NOW], [TODAY] :param param: parameter value @@ -151,7 +158,8 @@ def _replace_param_replacement(param, language): def _replace_param_transform_string(param): - """Transform param value according to the specified prefix + """ + Transform param value according to the specified prefix. Available transformations: DICT, LIST, INT, FLOAT, STR, UPPER, LOWER :param param: parameter value @@ -178,7 +186,8 @@ def _replace_param_transform_string(param): def _replace_param_date(param, language): - """Transform param value in a date after applying the specified delta + """ + Transform param value in a date after applying the specified delta. E.g. [TODAY - 2 DAYS], [NOW - 10 MINUTES] :param param: parameter value @@ -202,7 +211,8 @@ def _replace_param_date(param, language): def _replace_param_fixed_length(param): - """Generate a fixed length data element if param matches the expression [_WITH_LENGTH_] + """ + Generate a fixed length data element if param matches the expression [_WITH_LENGTH_] where can be: STRING, INTEGER, STRING_ARRAY, INTEGER_ARRAY, JSON. E.g. [STRING_WITH_LENGTH_15] @@ -259,3 +269,333 @@ def _infer_param_type(param): except Exception: pass return new_param + + +def map_param(param, context=None): + """ + Transform the given string by replacing specific patterns containing keys with their values, + which can be obtained from the Behave context or from environment files or variables. + See map_one_param function for a description of the available tags and replacement logic. + + :param param: string parameter + :param context: Behave context object + :return: string with the applied replacements + """ + if not isinstance(param, str): + return param + + map_regex = r"[\[CONF:|\[LANG:|\[POE:|\[ENV:|\[BASE64:|\[TOOLIUM:|\[CONTEXT:|\[FILE:][a-zA-Z\.\:\/\_\-\ 0-9]*\]" + map_expressions = re.compile(map_regex) + + # The parameter is just one config value + if map_expressions.split(param) == ['', '']: + return map_one_param(param, context) + + # The parameter is a combination of text and configuration parameters. + for match in map_expressions.findall(param): + param = param.replace(match, str(map_one_param(match, context))) + return param + + +def map_one_param(param, context=None): + """ + Analyze the pattern in the given string and find out its transformed value. + Available tags and replacement values: + [CONF:xxxx] Value from the config dict in context.project_config for the key xxxx (dot notation is used + for keys, e.g. key_1.key_2.0.key_3) + [LANG:xxxx] String from the texts dict in context.language_dict for the key xxxx, using the language + specified in context.language (dot notation is used for keys, e.g. button.label) + [POE:xxxx] Definition(s) from the POEditor terms list in context.poeditor_terms for the term xxxx + [TOOLIUM:xxxx] Value from the toolium config in context.toolium_config for the key xxxx (key format is + section_option, e.g. Driver_type) + [CONTEXT:xxxx] Value from the context storage dict for the key xxxx, or value of the context attribute xxxx, + if the former does not exist + [ENV:xxxx] Value of the OS environment variable xxxx + [FILE:xxxx] String with the content of the file in the path xxxx + [BASE64:xxxx] String with the base64 representation of the file content in the path xxxx + + :param param: string parameter + :param context: Behave context object + :return: transformed value or the original string if no transformation could be applied + """ + if not isinstance(param, str): + return param + + type, key = _get_mapping_type_and_key(param) + + mapping_functions = { + "CONF": { + "prerequisites": context and hasattr(context, "project_config"), + "function": map_json_param, + "args": [key, context.project_config if hasattr(context, "project_config") else None] + }, + "TOOLIUM": { + "prerequisites": context, + "function": map_toolium_param, + "args": [key, context] + }, + "CONTEXT": { + "prerequisites": context, + "function": get_value_from_context, + "args": [key, context] + }, + "LANG": { + "prerequisites": context, + "function": get_message_property, + "args": [key, context] + }, + "POE": { + "prerequisites": context, + "function": get_translation_by_poeditor_reference, + "args": [key, context] + }, + "ENV": { + "prerequisites": True, + "function": os.environ.get, + "args": [key] + }, + "FILE": { + "prerequisites": True, + "function": get_file, + "args": [key] + }, + "BASE64": { + "prerequisites": True, + "function": convert_file_to_base64, + "args": [key] + } + } + + if key and mapping_functions[type]["prerequisites"]: + return mapping_functions[type]["function"](*mapping_functions[type]["args"]) + else: + return param + + +def _get_mapping_type_and_key(param): + """ + Get the type and the key of the given string parameter to be mapped to the appropriate value. + + :param param: string parameter to be parsed + :return: a tuple with the type and the key to be mapped + """ + types = ["CONF", "LANG", "POE", "ENV", "BASE64", "TOOLIUM", "CONTEXT", "FILE"] + for type in types: + match_group = re.match(r"\[%s:(.*)\]" % type, param) + if match_group: + return type, match_group.group(1) + return None, None + + +def map_json_param(param, config, copy=True): + """ + Find the value of the given param using it as a key in the given dictionary. Dot notation is used, + so for example "service.vamps.user" could be used to retrieve the email in the following config example: + { + "services":{ + "vamps":{ + "user": "cyber-sec-user@11paths.com", + "password": "MyPassword" + } + } + } + + :param param: key to be searched (dot notation is used, e.g. "service.vamps.user"). + :param config: configuration dictionary + :param copy: boolean value to indicate whether to work with a copy of the given dictionary or not, + in which case, the dictionary content might be changed by this function (True by default) + :return: mapped value + """ + properties_list = param.split(".") + aux_config_json = deepcopy(config) if copy else config + try: + for property in properties_list: + if type(aux_config_json) is list: + aux_config_json = aux_config_json[int(property)] + else: + aux_config_json = aux_config_json[property] + + hidden_value = hide_passwords(param, aux_config_json) + logger.debug(f"Mapping param '{param}' to its configured value '{hidden_value}'") + except TypeError: + msg = f"Mapping chain not found in the given configuration dictionary. '{param}'" + logger.error(msg) + raise TypeError(msg) + except KeyError: + msg = f"Mapping chain not found in the given configuration dictionary. '{param}'" + logger.error(msg) + raise KeyError(msg) + except ValueError: + msg = f"Specified value is not a valid index. '{param}'" + logger.error(msg) + raise ValueError(msg) + except IndexError: + msg = f"Mapping index not found in the given configuration dictionary. '{param}'" + logger.error(msg) + raise IndexError(msg) + return os.path.expandvars(aux_config_json) \ + if aux_config_json and type(aux_config_json) not in [int, bool, float, list, dict] else aux_config_json + + +def hide_passwords(key, value): + """ + Return asterisks when the given key is a password that should be hidden. + + :param key: key name + :param value: value + :return: hidden value + """ + hidden_keys = ['key', 'pass', 'secret', 'code', 'token'] + hidden_value = '*****' + return hidden_value if any(hidden_key in key for hidden_key in hidden_keys) else value + + +def map_toolium_param(param, context): + """ + Find the value of the given param using it as a key in the current toolium configuration (context.toolium_config). + The param is expected to be in the form
_, so for example "TextExecution_environment" could be + used to retrieve the value of this toolium property (i.e. the string "QA"): + [TestExecution] + environment: QA + + :param param: key to be searched (e.g. "TextExecution_environment") + :param context: Behave context object + :return: mapped value + """ + try: + section = param.split("_", 1)[0] + property_name = param.split("_", 1)[1] + except IndexError: + msg = f"Invalid format in Toolium config param '{param}'. Valid format: 'Section_property'." + logger.error(msg) + raise IndexError(msg) + + try: + mapped_value = context.toolium_config.get(section, property_name) + logger.info(f"Mapping Toolium config param 'param' to its configured value '{mapped_value}'") + except Exception: + msg = f"'{param}' param not found in Toolium config file" + logger.error(msg) + raise Exception(msg) + return mapped_value + + +def get_value_from_context(param, context): + """ + Find the value of the given param using it as a key in the context storage dictionary (context.storage) or in the + context object itself. So for example, in the former case, "last_request_result" could be used to retrieve the value + from context.storage["last_request_result"], if it exists, whereas, in the latter case, "last_request.result" could + be used to retrieve the value from context.last_request.result, if it exists. + + :param param: key to be searched (e.g. "last_request_result" / "last_request.result") + :param context: Behave context object + :return: mapped value + """ + if context.storage and param in context.storage: + return context.storage[param] + logger.info(f"'{param}' key not found in context storage, searching in context") + try: + value = context + for part in param.split('.'): + value = getattr(value, part) + return value + except AttributeError: + msg = f"'{param}' not found neither in context storage nor in context" + logger.error(msg) + raise AttributeError(msg) + + +def get_message_property(param, context): + """ + Return the message for the given param, using it as a key in the list of language properties previously loaded + in the context (context.language_dict). Dot notation is used (e.g. "home.button.send"). + + :param param: message key + :param context: Behave context object + :return: the message mapped to the given key in the language set in the context (context.language) + """ + key_list = param.split(".") + language_dict_copy = deepcopy(context.language_dict) + try: + for key in key_list: + language_dict_copy = language_dict_copy[key] + logger.info(f"Mapping language param '{param}' to its configured value '{language_dict_copy[context.language]}'") + except KeyError: + msg = f"Mapping chain '{param}' not found in the language properties file" + logger.error(msg) + raise KeyError(msg) + + return language_dict_copy[context.language] + + +def get_translation_by_poeditor_reference(reference, context): + """ + Return the translation(s) for the given POEditor reference from the terms previously loaded in the context. + + :param reference: POEditor reference + :param context: Behave context object + :return: list of strings with the translations from POEditor or string with the translation if only one was found + """ + try: + context.poeditor_terms = context.poeditor_export + except AttributeError: + raise Exception("POEditor texts haven't been correctly downloaded!") + poeditor_conf = context.project_config['poeditor'] if 'poeditor' in context.project_config else {} + key = poeditor_conf['key_field'] if 'key_field' in poeditor_conf else 'reference' + search_type = poeditor_conf['search_type'] if 'search_type' in poeditor_conf else 'contains' + # Get POEditor prefixes and add no prefix option + poeditor_prefixes = poeditor_conf['prefixes'] if 'prefixes' in poeditor_conf else [] + poeditor_prefixes.append('') + for prefix in poeditor_prefixes: + if len(reference.split(':')) > 1 and prefix != '': + # If there are prefixes and the resource contains ':' apply prefix in the correct position + complete_reference = '%s:%s%s' % (reference.split(':')[0], prefix, reference.split(':')[1]) + else: + complete_reference = '%s%s' % (prefix, reference) + if search_type == 'exact': + translation = [term['definition'] for term in context.poeditor_terms + if complete_reference == term[key] and term['definition'] is not None] + else: + translation = [term['definition'] for term in context.poeditor_terms + if complete_reference in term[key] and term['definition'] is not None] + if len(translation) > 0: + break + assert len(translation) > 0, 'No translations found in POEditor for reference %s' % reference + translation = translation[0] if len(translation) == 1 else translation + return translation + + +def get_file(file_path): + """ + Return the content of a file given its path. + + :param file path: file path using slash as separator (e.g. "resources/files/doc.txt") + :return: string with the file content + """ + file_path_parts = file_path.split("/") + file_path = os.path.join(*file_path_parts) + if not os.path.exists(file_path): + raise Exception(f' ERROR - Cannot read file "{file_path}". Does not exist.') + + with open(file_path, 'r') as f: + return f.read() + + +def convert_file_to_base64(file_path): + """ + Return the content of a file given its path encoded in Base64. + + :param file path: file path using slash as separator (e.g. "resources/files/doc.txt") + :return: string with the file content encoded in Base64 + """ + file_path_parts = file_path.split("/") + file_path = os.path.join(*file_path_parts) + if not os.path.exists(file_path): + raise Exception(f' ERROR - Cannot read file "{file_path}". Does not exist.') + + try: + with open(file_path, "rb") as f: + file_content = base64.b64encode(f.read()).decode() + except Exception as e: + raise Exception(f' ERROR - converting the "{file_path}" file to Base64...: {e}') + return file_content diff --git a/toolium/utils/poeditor.py b/toolium/utils/poeditor.py new file mode 100644 index 00000000..2417a2bf --- /dev/null +++ b/toolium/utils/poeditor.py @@ -0,0 +1,333 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2022 Telefónica Investigación y Desarrollo, S.A.U. +This file is part of Toolium. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import json +import os +import time +import requests + +from configparser import NoOptionError +from urllib.request import URLopener +from toolium.utils.dataset import map_param +from toolium.driver_wrappers_pool import DriverWrappersPool + +""" +==================== +PROJECT REQUIREMENTS +==================== + +Set the language used to get the POEditor texts in the toolium config file ([TestExecution] language): + +[TestExecution] +language: es-es + +In your project configuration dictionary (context.project_config), add an entry like this: + +"poeditor": { + "base_url": "https://api.poeditor.com", + "api_token": "XXXXX", + "project_name": "My-Bot", + "prefixes": [], + "key_field": "reference", + "search_type": "contains", + "file_path": "output/poeditor_terms.json" +} + +If the file_path property is not configured as above, the file name will default to "poeditor_terms.json" +and the path will default to DriverWrappersPool.output_directory ("output" by default). + +NOTE: The api_token can be generated from POEditor in this url: https://poeditor.com/account/api +""" + +# POEDITOR ENDPOINTS + +ENDPOINT_POEDITOR_LIST_PROJECTS = "v2/projects/list" +ENDPOINT_POEDITOR_LIST_LANGUAGES = "v2/languages/list" +ENDPOINT_POEDITOR_LIST_TERMS = "v2/terms/list" +ENDPOINT_POEDITOR_EXPORT_PROJECT = "v2/projects/export" +ENDPOINT_POEDITOR_DOWNLOAD_FILE = "v2/download/file" + + +def download_poeditor_texts(context, file_type): + """ + Executes all steps to download texts from POEditor and saves them to a file in output dir + + :param context: behave context + :param file_type: only json supported in this first version + :return: N/A + """ + get_poeditor_project_info_by_name(context) + get_poeditor_language_codes(context) + export_poeditor_project(context, file_type) + save_downloaded_file(context) + + +def get_poeditor_project_info_by_name(context, project_name=None): + """ + Get POEditor project info from project name from config or parameter + + :param context: behave context + :param project_name: POEditor project name + :return: N/A (saves it to context.poeditor_project) + """ + projects = get_poeditor_projects(context) + project_name = project_name if project_name else map_param('[CONF:poeditor.project_name]', context) + projects_by_name = [project for project in projects if project['name'] == project_name] + + assert len(projects_by_name) == 1, "ERROR: Project name %s not found, available projects: %s" % \ + (project_name, [project['name'] for project in projects]) + context.poeditor_project = projects_by_name[0] + + +def get_poeditor_language_codes(context): + """ + Get language codes available for a given project ID + + :param context: behave context + :return: N/A (saves it to context.poeditor_language_list) + """ + params = {"api_token": get_poeditor_api_token(context), + "id": context.poeditor_project['id']} + + r = send_poeditor_request(context, ENDPOINT_POEDITOR_LIST_LANGUAGES, "POST", params, 200) + response_data = r.json() + assert_poeditor_response_code(response_data, "200") + + poeditor_language_list = [lang['code'] for lang in response_data['result']['languages']] + assert not len(poeditor_language_list) == 0, "ERROR: Not languages found in POEditor" + context.logger.info('POEditor languages in "%s" project: %s %s' % (context.poeditor_project['name'], + len(poeditor_language_list), + poeditor_language_list)) + + context.poeditor_language_list = poeditor_language_list + + +def search_terms_with_string(context, lang=None): + """ + Saves POEditor terms for a given existing language in that project + + :param context: behave context + :param lang: a valid language existing in that POEditor project + :return: N/A (saves it to context.poeditor_terms) + """ + lang = get_valid_lang(context, lang) + context.poeditor_terms = get_all_terms(context, lang) + + +def export_poeditor_project(context, file_type, lang=None): + """ + Export all texts in project to a given file type + + :param context: behave context + :param file_type: There are more available formats to download but only one is supported now: json + :param lang: if provided, should be a valid language configured in POEditor project + :return: N/A (saves it to context.poeditor_export) + """ + lang = get_valid_lang(context, lang) + assert file_type in ['json'], "Only json file type is supported at this moment" + context.poeditor_file_type = file_type + + params = {"api_token": get_poeditor_api_token(context), + "id": context.poeditor_project['id'], + "language": lang, + "type": file_type} + + r = send_poeditor_request(context, ENDPOINT_POEDITOR_EXPORT_PROJECT, "POST", params, 200) + response_data = r.json() + assert_poeditor_response_code(response_data, "200") + + context.poeditor_download_url = response_data['result']['url'] + filename = context.poeditor_download_url.split('/')[-1] + + r = send_poeditor_request(context, ENDPOINT_POEDITOR_DOWNLOAD_FILE + '/' + filename, "GET", {}, 200) + context.poeditor_export = r.json() + context.logger.info('POEditor terms in "%s" project with "%s" language: %s' % (context.poeditor_project['name'], + lang, len(context.poeditor_export))) + + +def save_downloaded_file(context): + """ + Saves POEditor terms to a file in output dir + + :param context: behave context + :return: N/A + """ + file_path = get_poeditor_file_path(context) + saved_file = URLopener() + saved_file.retrieve(context.poeditor_download_url, file_path) + context.logger.info('POEditor terms have been saved in "%s" file' % file_path) + + +def assert_poeditor_response_code(response_data, status_code): + """ + Check status code returned in POEditor response + + :param response_data: data received in poeditor API response as a dictionary + :param status_code: expected status code + """ + assert response_data['response']['code'] == status_code, f"{response_data['response']['code']} status code \ + has been received instead of {status_code} in POEditor response body. Response body: {response_data}" + + +def get_country_from_config_file(context): + """ + Gets the country to use later from config checking if it's a valid one in POEditor + + :param context: behave context + :return: country + """ + try: + country = context.toolium_config.get('TestExecution', 'language').lower() + except NoOptionError: + assert False, "There is no language configured in test, add it to config or use step with parameter lang_id" + + return country + + +def get_valid_lang(context, lang): + """ + Check if language provided is a valid one configured and returns the POEditor matched lang + + :param context: behave context + :param lang: a language from config or from lang parameter + :return: lang matched from POEditor + """ + lang = lang if lang else get_country_from_config_file(context) + if lang in context.poeditor_language_list: + matching_lang = lang + elif lang.split('-')[0] in context.poeditor_language_list: + matching_lang = lang.split('-')[0] + else: + assert False, "Language %s in config is not valid, use one of %s:" % (lang, context.poeditor_language_list) + return matching_lang + + +def get_poeditor_projects(context): + """ + Get the list of the projects configured in POEditor + + :param context: behave context + :return: the list of the projects + """ + params = {"api_token": get_poeditor_api_token(context)} + r = send_poeditor_request(context, ENDPOINT_POEDITOR_LIST_PROJECTS, "POST", params, 200) + response_data = r.json() + assert_poeditor_response_code(response_data, "200") + projects = response_data['result']['projects'] + projects_names = [project['name'] for project in projects] + context.logger.info('POEditor projects: %s %s' % (len(projects_names), projects_names)) + return projects + + +def send_poeditor_request(context, endpoint, method, params, status_code): + """ + Send a request to the POEditor API + + :param context: behave context + :param endpoint: endpoint path + :param method: HTTP method to be used in the request + :param params: parameters to be sent in the request + :param code: expected status code + :return: response + """ + url = "/".join([map_param('[CONF:poeditor.base_url]', context), endpoint]) + r = requests.request(method, url, data=params) + assert r.status_code == status_code, f"{r.status_code} status code has been received instead of {status_code} \ + in POEditor response. Response body: {r.json()}" + return r + + +def get_all_terms(context, lang): + """ + Get all terms for a given language configured in POEditor + + :param context: behave context + :param lang: a valid language configured in POEditor project + :return: the list of terms + """ + params = {"api_token": get_poeditor_api_token(context), + "id": context.poeditor_project['id'], + "language": lang} + + r = send_poeditor_request(context, ENDPOINT_POEDITOR_LIST_TERMS, "POST", params, 200) + response_data = r.json() + assert_poeditor_response_code(response_data, "200") + terms = response_data['result']['terms'] + context.logger.info('POEditor terms in "%s" project with "%s" language: %s' % (context.poeditor_project['name'], + lang, len(terms))) + + return terms + + +def load_poeditor_texts(context): + """ + Download POEditor texts and save in output folder if the config exists or use previously downloaded texts + + :param context: behave context + """ + if get_poeditor_api_token(context): + # Try to get poeditor mode param from toolium config first + poeditor_mode = context.toolium_config.get_optional('TestExecution', 'poeditor_mode') + if poeditor_mode: + context.project_config['poeditor']['mode'] = poeditor_mode + if 'mode' in context.project_config['poeditor'] and map_param('[CONF:poeditor.mode]', context) == 'offline': + file_path = get_poeditor_file_path(context) + + # With offline POEditor mode, file must exist + if not os.path.exists(file_path): + error_message = 'You are using offline POEditor mode but poeditor file has not been found in %s' % \ + file_path + context.logger.error(error_message) + assert False, error_message + + with open(file_path, 'r') as f: + context.poeditor_export = json.load(f) + last_mod_time = time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(os.path.getmtime(file_path))) + context.logger.info('Using local POEditor file "%s" with date: %s' % (file_path, last_mod_time)) + else: # without mode configured or mode = 'online' + download_poeditor_texts(context, 'json') + else: + context.logger.info("POEditor is not configured") + + +def get_poeditor_file_path(context): + """ + Get POEditor file path + + :param context: behave context + :return: poeditor file path + """ + try: + file_path = context.project_config['poeditor']['file_path'] + except KeyError: + file_type = context.poeditor_file_type if hasattr(context, 'poeditor_file_type') else 'json' + file_path = os.path.join(DriverWrappersPool.output_directory, 'poeditor_terms.%s' % file_type) + return file_path + + +def get_poeditor_api_token(context): + """ + Get POEditor api token from environment property or configuration property + + :return: poeditor api token + """ + try: + api_token = os.environ['poeditor_api_token'] + except KeyError: + api_token = map_param('[CONF:poeditor.api_token]', context) + return api_token