From 9939c0a34288e14d89e0ac2070a0bb72b14cd7ab Mon Sep 17 00:00:00 2001 From: robcxyz Date: Sat, 23 Dec 2023 18:06:18 -0800 Subject: [PATCH] refactorchange yamlencode/yamldecode to yaml_encode/yaml_decode and add same hooks to yaml, toml, and ini providers --- .github/workflows/docs.yml | 2 +- providers/ini/hooks/ini.py | 50 +++++++++++++++++++ providers/ini/tests/test_provider_ini.py | 25 ++++++++++ providers/json/hooks/jsons.py | 36 ++++++------- providers/json/tests/test_provider_json.py | 13 +++++ providers/toml/.tackle.meta.yaml | 13 +---- providers/toml/hooks/tomls.py | 49 +++++++++--------- providers/toml/requirements.txt | 1 - providers/toml/tests/test_provider_toml.py | 16 ++---- providers/toml/tests/write.yaml | 12 ----- providers/yaml/hooks/yaml_in_place.py | 6 +-- providers/yaml/hooks/yamls.py | 18 +++---- .../yaml/tests/test_provider_system_yaml.py | 4 +- providers/yaml/tests/yaml_decode.yaml | 6 +++ .../{yamlencode.yaml => yaml_encode.yaml} | 2 +- providers/yaml/tests/yamldecode.yaml | 6 --- 16 files changed, 152 insertions(+), 107 deletions(-) delete mode 100644 providers/toml/requirements.txt delete mode 100644 providers/toml/tests/write.yaml create mode 100644 providers/yaml/tests/yaml_decode.yaml rename providers/yaml/tests/{yamlencode.yaml => yaml_encode.yaml} (63%) delete mode 100644 providers/yaml/tests/yamldecode.yaml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2339c0f40..f74ceb3ec 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -19,7 +19,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -e . - pip install -r docs/requirements.txt + pip install -r requirements-docs.txt pip install -e .[all] - name: Build docs diff --git a/providers/ini/hooks/ini.py b/providers/ini/hooks/ini.py index 06d4e5bc9..e85b56c77 100644 --- a/providers/ini/hooks/ini.py +++ b/providers/ini/hooks/ini.py @@ -1,5 +1,6 @@ import configparser import os +from io import StringIO from typing import Union from tackle import BaseHook, Field @@ -53,3 +54,52 @@ def exec(self) -> Union[dict, str, list]: output[section][option] = config.get(section, option) return output + + +class IniEncodeHook(BaseHook): + """Hook for converting a dict to an ini encoded string.""" + + hook_name: str = 'ini_encode' + data: dict | str = Field( + ..., + description="Map or renderable string to data to convert to ini string.", + render_by_default=True, + ) + args: list = ['data'] + + def exec(self) -> str: + if not isinstance(self.data, dict): + raise ValueError("INI serialization requires a dictionary input") + + config = configparser.ConfigParser() + # https://stackoverflow.com/a/19359720/12642712 + config.optionxform = str + + for section, values in self.data.items(): + config[section] = values + + with StringIO() as string_stream: + config.write(string_stream) + return string_stream.getvalue() + + +class IniDecodeHook(BaseHook): + """Hook for decoding an ini string to a dict.""" + + hook_name: str = 'ini_decode' + data: str = Field( + ..., + description="Yaml string to convert to dict.", + ) + args: list = ['data'] + + def exec(self) -> dict: + config = configparser.ConfigParser() + # https://stackoverflow.com/a/19359720/12642712 + config.optionxform = str + string_stream = StringIO(self.data) + config.read_file(string_stream) + + # Convert to a regular dictionary + output = {section: dict(config.items(section)) for section in config.sections()} + return output diff --git a/providers/ini/tests/test_provider_ini.py b/providers/ini/tests/test_provider_ini.py index ff802cc91..3793ed800 100644 --- a/providers/ini/tests/test_provider_ini.py +++ b/providers/ini/tests/test_provider_ini.py @@ -3,6 +3,7 @@ import pytest from tackle.main import tackle +from tackle.utils.hooks import get_hook @pytest.fixture() @@ -22,3 +23,27 @@ def test_provider_ini_read(clean_outputs): assert os.path.exists('output.ini') assert output['read']['section']['stuff'] == 'things' assert output['read']['another']['foo'] is None + + +def test_provider_ini_encode(): + Hook = get_hook('ini_encode') + output = Hook(data={"Section1": {"keyA": "valueA", "keyB": "valueB"}}).exec() + assert ( + output + == """[Section1] +keyA = valueA +keyB = valueB + +""" + ) + + +def test_provider_ini_decode(): + Hook = get_hook('ini_decode') + output = Hook( + data="""[Section1] +keyA = valueA +keyB = valueB +""" + ).exec() + assert output == {"Section1": {"keyA": "valueA", "keyB": "valueB"}} diff --git a/providers/json/hooks/jsons.py b/providers/json/hooks/jsons.py index 980365f5d..79dc82e5e 100644 --- a/providers/json/hooks/jsons.py +++ b/providers/json/hooks/jsons.py @@ -36,31 +36,27 @@ def exec(self) -> Union[dict, str]: return data -class JsonifyHook(BaseHook): - """ - Hook for reading and writing json. Hook reads from `path` if no `data` field is - provided, otherwise it writes the `data` to `path`. - """ +class JsonEncodeHook(BaseHook): + """Hook for converting a dict to a JSON encoded string.""" - hook_name: str = 'jsonify' + hook_name: str = 'json_encode' data: Union[dict, list, str] = Field( ..., - description="Map/list or renderable string to a map/list key to write.", + description="Map/list or renderable string to data to convert to JSON string.", render_by_default=True, ) args: list = ['data'] - def exec(self) -> Union[dict, str]: + def exec(self) -> str: return json.dumps(self.data) - # if self.path: - # self.path = os.path.abspath( - # os.path.expanduser(os.path.expandvars(self.path)) - # ) - # # Make path if it does not exist - # if not os.path.exists(os.path.dirname(self.path)) and self.data: - # os.makedirs(os.path.dirname(self.path)) - # with open(self.path, 'w') as f: - # json.dump(self.data, f) - # return self.path - # else: - # return json.dumps(self.data) + + +class JsonDecodeHook(BaseHook): + """Hook for decoding a JSON string to a dict.""" + + hook_name: str = 'json_decode' + data: str = Field(..., description="JSON string to convert to dict.") + args: list = ['data'] + + def exec(self) -> Union[dict, list]: + return json.loads(self.data) diff --git a/providers/json/tests/test_provider_json.py b/providers/json/tests/test_provider_json.py index 22d0f31f6..666f689bf 100644 --- a/providers/json/tests/test_provider_json.py +++ b/providers/json/tests/test_provider_json.py @@ -1,6 +1,19 @@ from tackle import tackle +from tackle.utils.hooks import get_hook def test_provider_json(): output = tackle('json.yaml') assert output + + +def test_provider_json_encode(): + Hook = get_hook('json_encode') + output = Hook(data={"Section1": {"keyA": "valueA", "keyB": "valueB"}}).exec() + assert output == '{"Section1": {"keyA": "valueA", "keyB": "valueB"}}' + + +def test_provider_json_decode(): + Hook = get_hook('json_decode') + output = Hook(data='{"Section1": {"keyA": "valueA", "keyB": "valueB"}}').exec() + assert output == {"Section1": {"keyA": "valueA", "keyB": "valueB"}} diff --git a/providers/toml/.tackle.meta.yaml b/providers/toml/.tackle.meta.yaml index 10ea38d25..87094853d 100644 --- a/providers/toml/.tackle.meta.yaml +++ b/providers/toml/.tackle.meta.yaml @@ -1,7 +1,8 @@ name: TOML Provider -description: Reading and writing toml files. +description: | + Reading toml files. For writing toml you will need a third party provider since this wraps python's native toml library which only supports reads. examples: [] @@ -14,13 +15,3 @@ hook_examples: ->: toml path: path/to/toml/file.toml compact->: toml path/to/toml/file.toml - - name: Write toml - description: Write a toml file from a key - content: | - stuff: - and: things - expanded: - ->: toml - path: path/to/toml/file.toml - contents: "{{ stuff }}" - compact->: toml path/to/toml/file.toml "{{ this }}" diff --git a/providers/toml/hooks/tomls.py b/providers/toml/hooks/tomls.py index 8969cfd87..8faf239a7 100644 --- a/providers/toml/hooks/tomls.py +++ b/providers/toml/hooks/tomls.py @@ -1,38 +1,35 @@ -try: - import tomli as toml -except ModuleNotFoundError: - import tomllib as toml - import os -from typing import MutableMapping, Union +from typing import Union + +import tomli as toml from tackle import BaseHook, Field class TomlHook(BaseHook): - """Hook for reading toml. Does not support write which needs another provider""" + """ + Hook for reading TOML. Wraps python's native toml library which does not support + writing toml, only reading. + """ hook_name: str = 'toml' - path: str = Field(..., description="The file path to put read or write to.") - data: Union[dict, list, str] = Field( - None, - description="Map/list or renderable string to a map/list key to write.", - render_by_default=True, - ) - args: list = ['path', 'data'] - - def exec(self) -> Union[MutableMapping, str]: + path: str = Field(..., description="The file path to read or write to.") + + args: list = ['path'] + + def exec(self) -> Union[dict, str]: self.path = os.path.abspath(os.path.expanduser(os.path.expandvars(self.path))) - # Make path if it does not exist - # if not os.path.exists(os.path.dirname(self.path)) and self.data: - # os.makedirs(os.path.dirname(self.path)) - - # if self.data: - # with open(self.path, 'w') as f: - # toml.dump(self.data, f) - # return self.path - # - # else: with open(self.path, 'rb') as f: data = toml.load(f) return data + + +class TomlDecodeHook(BaseHook): + """Hook for decoding a TOML string to a dict.""" + + hook_name: str = 'toml_decode' + data: str = Field(..., description="TOML string to convert to dict.") + args: list = ['data'] + + def exec(self) -> dict: + return toml.loads(self.data) diff --git a/providers/toml/requirements.txt b/providers/toml/requirements.txt deleted file mode 100644 index fd34bffe6..000000000 --- a/providers/toml/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -toml>=0.10.0 \ No newline at end of file diff --git a/providers/toml/tests/test_provider_toml.py b/providers/toml/tests/test_provider_toml.py index fbd9e32aa..9fc8ad78a 100644 --- a/providers/toml/tests/test_provider_toml.py +++ b/providers/toml/tests/test_provider_toml.py @@ -1,4 +1,5 @@ from tackle.main import tackle +from tackle.utils.hooks import get_hook def test_provider_toml_hook_read(): @@ -6,14 +7,7 @@ def test_provider_toml_hook_read(): assert 'owner' in o['read'].keys() -# @pytest.fixture() -# def clean_toml(): -# """Remove outputs.""" -# yield -# if os.path.exists('writing.toml'): -# os.remove('writing.toml') -# -# -# def test_provider_toml_hook_write(clean_toml): -# tackle('write.yaml', no_input=True) -# assert os.path.exists('writing.toml') +def test_provider_toml_decode(): + Hook = get_hook('toml_decode') + output = Hook(data='[Section1]\nkeyA = "valueA"\nkeyB = "valueB"\n').exec() + assert output == {"Section1": {"keyA": "valueA", "keyB": "valueB"}} diff --git a/providers/toml/tests/write.yaml b/providers/toml/tests/write.yaml deleted file mode 100644 index cd96f90ce..000000000 --- a/providers/toml/tests/write.yaml +++ /dev/null @@ -1,12 +0,0 @@ -input: - ->: var - input: - foo: bar - baz: - - bing - - ding - -write: - ->: toml - path: writing.toml - data: "{{ input }}" diff --git a/providers/yaml/hooks/yaml_in_place.py b/providers/yaml/hooks/yaml_in_place.py index 2822d5c5d..59304a9d1 100644 --- a/providers/yaml/hooks/yaml_in_place.py +++ b/providers/yaml/hooks/yaml_in_place.py @@ -1,7 +1,3 @@ -""" -TODO: https://github.com/sudoblockio/tackle/issues/100 - Should change -""" import os import re from typing import Any, Dict, List, Union @@ -15,7 +11,7 @@ class YamlHook(BaseHook): """ Hook for modifying a yaml in place (ie read, transform, and write back to the file - in one operation). WIP -> Contributions welcome. + in one operation). WIP -> https://github.com/sudoblockio/tackle/issues/100. """ hook_name: str = 'yaml_in_place' diff --git a/providers/yaml/hooks/yamls.py b/providers/yaml/hooks/yamls.py index 3a1b0fd58..3a6acafd5 100644 --- a/providers/yaml/hooks/yamls.py +++ b/providers/yaml/hooks/yamls.py @@ -1,4 +1,5 @@ import os +from io import StringIO from typing import Union from ruyaml import YAML @@ -55,7 +56,7 @@ def exec(self) -> Union[dict, str]: class YamlEncodeHook(BaseHook): """Hook for converting a dict to a yaml encoded string.""" - hook_name: str = 'yamlencode' + hook_name: str = 'yaml_encode' data: Union[dict, list, str] = Field( ..., description="Map/list or renderable string to data to convert to yaml string.", @@ -63,22 +64,17 @@ class YamlEncodeHook(BaseHook): ) args: list = ['data'] - def exec(self) -> Union[dict, str]: - from io import StringIO - + def exec(self) -> str: yaml = YAML() - options = {} - string_stream = StringIO() - yaml.dump(self.data, string_stream, **options) - output_str = string_stream.getvalue() - string_stream.close() - return output_str + with StringIO() as string_stream: + yaml.dump(self.data, string_stream) + return string_stream.getvalue() class YamlDecodeHook(BaseHook): """Hook for decoding a yaml string to a dict.""" - hook_name: str = 'yamldecode' + hook_name: str = 'yaml_decode' data: str = Field(..., description="Yaml string to convert to dict.") args: list = ['data'] diff --git a/providers/yaml/tests/test_provider_system_yaml.py b/providers/yaml/tests/test_provider_system_yaml.py index 6ad9f5654..c2cc5c6ad 100644 --- a/providers/yaml/tests/test_provider_system_yaml.py +++ b/providers/yaml/tests/test_provider_system_yaml.py @@ -101,12 +101,12 @@ def test_provider_system_hook_yaml_append(clean_outputs): def test_yaml_yamlify(clean_outputs): - output = tackle('yamlencode.yaml', no_input=True) + output = tackle('yaml_encode.yaml', no_input=True) assert isinstance(output['out'], str) assert 'stuff: things' in output['out'] def test_yaml_yamldecode(clean_outputs): - output = tackle('yamldecode.yaml', no_input=True) + output = tackle('yaml_decode.yaml', no_input=True) assert isinstance(output['out'], dict) assert output['out']['stuff'] == 'things' diff --git a/providers/yaml/tests/yaml_decode.yaml b/providers/yaml/tests/yaml_decode.yaml new file mode 100644 index 000000000..1870bf8a8 --- /dev/null +++ b/providers/yaml/tests/yaml_decode.yaml @@ -0,0 +1,6 @@ +in->: file read.yaml + +#out_filter->: "{{in|yamldecode}}" +out_function->: "{{yaml_decode(in)['stuff']}}" + +out->: yaml_decode {{in}} diff --git a/providers/yaml/tests/yamlencode.yaml b/providers/yaml/tests/yaml_encode.yaml similarity index 63% rename from providers/yaml/tests/yamlencode.yaml rename to providers/yaml/tests/yaml_encode.yaml index 9e81e2fa7..d786af771 100644 --- a/providers/yaml/tests/yamlencode.yaml +++ b/providers/yaml/tests/yaml_encode.yaml @@ -1,4 +1,4 @@ in->: yaml read.yaml #out->: "{{in|yamlencode}}" -out->: "{{yamlencode(in)}}" +out->: "{{yaml_encode(in)}}" diff --git a/providers/yaml/tests/yamldecode.yaml b/providers/yaml/tests/yamldecode.yaml deleted file mode 100644 index 29a3bca7e..000000000 --- a/providers/yaml/tests/yamldecode.yaml +++ /dev/null @@ -1,6 +0,0 @@ -in->: file read.yaml - -#out_filter->: "{{in|yamldecode}}" -out_function->: "{{yamldecode(in)['stuff']}}" - -out->: yamldecode {{in}}