From 2d7a3183d9b1381f70e58d47301dbfe31f9edb3a Mon Sep 17 00:00:00 2001 From: Sanhe Hu Date: Wed, 17 May 2023 03:36:48 -0400 Subject: [PATCH] refactor multi_env_json --- config_patterns/patterns/multi_env_json.py | 107 +----- example/multi_env_json/config.json | 22 +- example/multi_env_json/config_define.py | 4 + example/multi_env_json/secret_config.json | 20 +- .../multi_env_json/test_multi_env_json.py | 353 ------------------ 5 files changed, 26 insertions(+), 480 deletions(-) diff --git a/config_patterns/patterns/multi_env_json.py b/config_patterns/patterns/multi_env_json.py index 91d3c55..1d97d86 100644 --- a/config_patterns/patterns/multi_env_json.py +++ b/config_patterns/patterns/multi_env_json.py @@ -27,7 +27,9 @@ from ..logger import logger from ..jsonutils import json_loads from ..compat import cached_property -from ..vendor.strutils import camel2under, slugify +from ..vendor.strutils import slugify +from .hierarchy import apply_shared_value +from .merge_key_value import merge_key_value def validate_project_name(project_name: str): @@ -79,109 +81,6 @@ def normalize_parameter_name(param_name: str) -> str: return param_name -SHARED = "_shared" - - -def set_shared_value( - path: str, - value: T.Any, - data: T.Union[list, dict], -): - """ - Set shared value to all items in a list or dict. - """ - parts = path.split(".") - if len(parts) == 1: - if isinstance(data, dict): - data.setdefault(parts[0], value) - elif isinstance(data, list): - for item in data: - item.setdefault(parts[0], value) - else: # pragma: no cover - raise NotImplementedError - return - key = parts[0] - if key == "*": - for k, v in data.items(): - if k != SHARED: - set_shared_value( - path=".".join(parts[1:]), - value=value, - data=v, - ) - else: - if isinstance(data, dict): - set_shared_value( - path=".".join(parts[1:]), - value=value, - data=data[key], - ) - elif isinstance(data, list): - for item in data: - set_shared_value( - path=".".join(parts[1:]), - value=value, - data=item[key], - ) - else: # pragma: no cover - raise NotImplementedError - - -def apply_shared_value(data: dict): - # implement recursion pattern - for key, value in data.items(): - if key == SHARED: - continue - if isinstance(value, dict): - apply_shared_value(value) - if isinstance(value, list): - for item in value: - if isinstance(item, dict): - apply_shared_value(item) - - has_shared = SHARED in data - if has_shared is False: - return - - shared_data = data.pop(SHARED) - for path, value in shared_data.items(): - set_shared_value(path=path, value=value, data=data) - - -def merge_key_value(data1: dict, data2: dict): - data1 = copy.deepcopy(data1) - data2 = copy.deepcopy(data2) - - difference = data2.keys() - data1.keys() - intersection = data1.keys() & data2.keys() - - for key in difference: - data1[key] = data2[key] - - for key in intersection: - value1, value2 = data1[key], data2[key] - if isinstance(value1, dict) and isinstance(value2, dict): - data1[key] = merge_key_value(value1, value2) - elif isinstance(value1, list) and isinstance(value2, list): - if len(value1) != len(value2): - raise ValueError(f"list length mismatch: key = {key!r}") - value = list() - for item1, item2 in zip(value1, value2): - if isinstance(item1, dict) and isinstance(item2, dict): - value.append(merge_key_value(item1, item2)) - else: - raise ValueError - data1[key] = value - else: - raise TypeError( - f"type of data1[{key!r}] and type of data2[{key!r}] " - f"has to be both dict or list of dict to merge! " - f"they are {type(value1)} and {type(value2)}." - ) - - return data1 - - @dataclasses.dataclass class BaseEnv: """ diff --git a/example/multi_env_json/config.json b/example/multi_env_json/config.json index c2f6761..be36cc9 100644 --- a/example/multi_env_json/config.json +++ b/example/multi_env_json/config.json @@ -1,16 +1,14 @@ { - "shared": { - "project_name": "my_project" + "_shared": { + "*.project_name": "my_project" }, - "envs": { - "dev": { - "username": "dev.user" - }, - "int": { - "username": "int.user" - }, - "prod": { - "username": "prod.user" - } + "dev": { + "username": "dev.user" + }, + "int": { + "username": "int.user" + }, + "prod": { + "username": "prod.user" } } \ No newline at end of file diff --git a/example/multi_env_json/config_define.py b/example/multi_env_json/config_define.py index 314e1c6..26371f4 100644 --- a/example/multi_env_json/config_define.py +++ b/example/multi_env_json/config_define.py @@ -27,6 +27,10 @@ class Env(BaseEnv): def parameter_name(self) -> str: return f"/app/{normalize_parameter_name(self.prefix_name_snake)}" + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + @dataclasses.dataclass class Config(BaseConfig): diff --git a/example/multi_env_json/secret_config.json b/example/multi_env_json/secret_config.json index 26dbaf7..67b2391 100644 --- a/example/multi_env_json/secret_config.json +++ b/example/multi_env_json/secret_config.json @@ -1,15 +1,13 @@ { - "shared": { + "_shared": { }, - "envs": { - "dev": { - "password": "dev.password" - }, - "int": { - "password": "int.password" - }, - "prod": { - "password": "prod.password" - } + "dev": { + "password": "dev.password" + }, + "int": { + "password": "int.password" + }, + "prod": { + "password": "prod.password" } } \ No newline at end of file diff --git a/tests/patterns/multi_env_json/test_multi_env_json.py b/tests/patterns/multi_env_json/test_multi_env_json.py index 0a488be..6e8c565 100644 --- a/tests/patterns/multi_env_json/test_multi_env_json.py +++ b/tests/patterns/multi_env_json/test_multi_env_json.py @@ -2,21 +2,16 @@ import typing as T import pytest -import copy import json import dataclasses from pathlib import Path import moto from boto_session_manager import BotoSesManager -from rich import print as rprint from config_patterns.patterns.multi_env_json import ( validate_project_name, validate_env_name, - set_shared_value, - apply_shared_value, - merge_key_value, normalize_parameter_name, BaseEnvEnum, BaseEnv, @@ -151,354 +146,6 @@ def test_normalize_parameter_name(): assert normalize_parameter_name("ssm-project") == "p-ssm-project" -def _test_set_shared_value_multi_parts_path(): - data = {"key1": "value1"} - set_shared_value(path="key1", value="invalid", data=data) - assert data["key1"] == "value1" - set_shared_value(path="key2", value="value2", data=data) - assert data["key2"] == "value2" - - data = {"key1": {"key11": "value11"}} - set_shared_value(path="key1.key11", value="invalid", data=data) - assert data["key1"]["key11"] == "value11" - set_shared_value(path="key1.key12", value="value12", data=data) - assert data["key1"]["key12"] == "value12" - - data = {"key1": {"key11": {"key111": "value111"}}} - set_shared_value(path="key1.key11.key111", value="invalid", data=data) - assert data["key1"]["key11"]["key111"] == "value111" - set_shared_value(path="key1.key11.key112", value="value112", data=data) - assert data["key1"]["key11"]["key112"] == "value112" - - -def _test_set_shared_value_list(): - data = [ - {"key1": "value1"}, - {"key1": "value1"}, - ] - set_shared_value(path="key1", value="invalid", data=data) - assert data == [{"key1": "value1"}, {"key1": "value1"}] - set_shared_value(path="key2", value="value2", data=data) - assert data == [ - {"key1": "value1", "key2": "value2"}, - {"key1": "value1", "key2": "value2"}, - ] - - data = { - "tags": [ - {"key1": "value1"}, - {"key1": "value1"}, - ], - } - set_shared_value(path="tags.key1", value="invalid", data=data) - assert data["tags"] == [{"key1": "value1"}, {"key1": "value1"}] - set_shared_value(path="tags.key2", value="value2", data=data) - assert data["tags"] == [ - {"key1": "value1", "key2": "value2"}, - {"key1": "value1", "key2": "value2"}, - ] - - data = { - "persons": [ - { - "name": "alice", - "tags": [ - {"key1": "value1"}, - {"key1": "value1"}, - ], - }, - { - "name": "bob", - "tags": [ - {"key1": "value1"}, - {"key1": "value1"}, - ], - }, - ], - } - set_shared_value(path="persons.tags.key2", value="value2", data=data) - assert data["persons"][0]["tags"] == [ - {"key1": "value1", "key2": "value2"}, - {"key1": "value1", "key2": "value2"}, - ] - assert data["persons"][1]["tags"] == [ - {"key1": "value1", "key2": "value2"}, - {"key1": "value1", "key2": "value2"}, - ] - - -def _test_set_shared_value_star_notation(): - data = { - "dev": {"key1": "dev_value1"}, - "prod": {"key1": "prod_value1"}, - } - set_shared_value(path="*.key1", value="invalid", data=data) - assert data["dev"]["key1"] == "dev_value1" - assert data["prod"]["key1"] == "prod_value1" - set_shared_value(path="*.key2", value="value2", data=data) - assert data["dev"]["key2"] == "value2" - assert data["prod"]["key2"] == "value2" - - data = { - "envs": { - "dev": {"key1": "dev_value1"}, - "prod": {"key1": "prod_value1"}, - } - } - set_shared_value(path="envs.*.key1", value="invalid", data=data) - assert data["envs"]["dev"]["key1"] == "dev_value1" - assert data["envs"]["prod"]["key1"] == "prod_value1" - set_shared_value(path="envs.*.key2", value="value2", data=data) - assert data["envs"]["dev"]["key2"] == "value2" - assert data["envs"]["prod"]["key2"] == "value2" - - raw_data = { - "envs": { - "dev": { - "server": { - "blue": {"key1": "dev_blue_value1"}, - "green": {"key1": "dev_green_value1"}, - } - }, - "prod": { - "server": { - "black": {"key1": "prod_black_value1"}, - "white": {"key1": "prod_white_value1"}, - } - }, - } - } - - data = copy.deepcopy(raw_data) - set_shared_value(path="envs.*.server.*.key1", value="value2", data=data) - assert data["envs"]["dev"]["server"]["blue"]["key1"] == "dev_blue_value1" - assert data["envs"]["dev"]["server"]["green"]["key1"] == "dev_green_value1" - assert data["envs"]["prod"]["server"]["black"]["key1"] == "prod_black_value1" - assert data["envs"]["prod"]["server"]["white"]["key1"] == "prod_white_value1" - set_shared_value(path="envs.*.server.*.key2", value="value2", data=data) - assert data["envs"]["dev"]["server"]["blue"]["key2"] == "value2" - assert data["envs"]["dev"]["server"]["green"]["key2"] == "value2" - assert data["envs"]["prod"]["server"]["black"]["key2"] == "value2" - assert data["envs"]["prod"]["server"]["white"]["key2"] == "value2" - - data = copy.deepcopy(raw_data) - set_shared_value(path="envs.dev.server.*.key1", value="value2", data=data) - assert data["envs"]["dev"]["server"]["blue"]["key1"] == "dev_blue_value1" - assert data["envs"]["dev"]["server"]["green"]["key1"] == "dev_green_value1" - assert data["envs"]["prod"]["server"]["black"]["key1"] == "prod_black_value1" - assert data["envs"]["prod"]["server"]["white"]["key1"] == "prod_white_value1" - set_shared_value(path="envs.dev.server.*.key2", value="value2", data=data) - assert data["envs"]["dev"]["server"]["blue"]["key2"] == "value2" - assert data["envs"]["dev"]["server"]["green"]["key2"] == "value2" - assert "key2" not in data["envs"]["prod"]["server"]["black"] - assert "key2" not in data["envs"]["prod"]["server"]["white"] - - raw_data = { - "envs": { - "dev": { - "server": { - "blue": { - "tags": [ - {"key1": "dev_blue_value1"}, - ], - }, - "green": { - "tags": [ - {"key1": "dev_green_value1"}, - ], - }, - } - }, - "prod": { - "server": { - "black": { - "tags": [ - {"key1": "prod_black_value1"}, - ], - }, - "white": { - "tags": [ - {"key1": "prod_white_value1"}, - ], - }, - } - }, - } - } - data = copy.deepcopy(raw_data) - set_shared_value(path="envs.*.server.*.tags.key1", value="invalid", data=data) - assert data["envs"]["dev"]["server"]["blue"]["tags"][0]["key1"] == "dev_blue_value1" - assert ( - data["envs"]["dev"]["server"]["green"]["tags"][0]["key1"] == "dev_green_value1" - ) - assert ( - data["envs"]["prod"]["server"]["black"]["tags"][0]["key1"] - == "prod_black_value1" - ) - assert ( - data["envs"]["prod"]["server"]["white"]["tags"][0]["key1"] - == "prod_white_value1" - ) - set_shared_value(path="envs.*.server.*.tags.key2", value="value2", data=data) - assert data["envs"]["dev"]["server"]["blue"]["tags"][0]["key2"] == "value2" - assert data["envs"]["dev"]["server"]["green"]["tags"][0]["key2"] == "value2" - assert data["envs"]["prod"]["server"]["black"]["tags"][0]["key2"] == "value2" - assert data["envs"]["prod"]["server"]["white"]["tags"][0]["key2"] == "value2" - - -def test_set_shared_value(): - _test_set_shared_value_multi_parts_path() - _test_set_shared_value_list() - _test_set_shared_value_star_notation() - - -def test_apply_shared_value(): - data = { - "_shared": { - "*.key2": "value2", - "*.a_dict.key2": "value2", - "*.servers.*.cpu": 1, - "*.databases.port": 1, - }, - "dev": { - "key1": "dev_value1", - "a_dict": { - "key1": "dev_value1", - }, - "servers": { - "_shared": { - "*.cpu": 2, - }, - "blue": {}, - "green": {"cpu": 4}, - }, - "databases": [ - {"host": "db1.com"}, - {"host": "db2.com", "port": 2}, - ], - }, - "prod": { - "_shared": { - "databases.port": 3, - }, - "key1": "prod_value1", - "a_dict": { - "key1": "prod_value1", - }, - "servers": { - "black": {}, - "white": {"cpu": 8}, - }, - "databases": [ - {"host": "db3.com"}, - {"host": "db4.com", "port": 4}, - ], - }, - } - apply_shared_value(data) - assert data == { - "dev": { - "key1": "dev_value1", - "key2": "value2", - "a_dict": { - "key1": "dev_value1", - "key2": "value2", - }, - "servers": { - "blue": {"cpu": 2}, - "green": {"cpu": 4}, - }, - "databases": [ - {"host": "db1.com", "port": 1}, - {"host": "db2.com", "port": 2}, - ], - }, - "prod": { - "key1": "prod_value1", - "key2": "value2", - "a_dict": { - "key1": "prod_value1", - "key2": "value2", - }, - "servers": { - "black": {"cpu": 1}, - "white": {"cpu": 8}, - }, - "databases": [ - {"host": "db3.com", "port": 3}, - {"host": "db4.com", "port": 4}, - ], - }, - } - - -def test_merge_key_value(): - data1 = { - "dev": { - "username": "dev.user", - }, - "test": { - "username": "test.user", - "server": { - "username": "ubuntu", - }, - "databases": [ - {"host": "www.db1.com", "username": "admin"}, - {"host": "www.db2.com", "username": "admin"}, - ], - }, - } - data2 = { - "test": { - "password": "test.password", - "server": { - "password": "ubuntu.password", - }, - "databases": [ - {"password": "db1pwd"}, - {"password": "db2pwd"}, - ], - }, - "prod": { - "password": "prod.password", - }, - } - data = merge_key_value(data1, data2) - assert data == { - "dev": {"username": "dev.user"}, - "test": { - "username": "test.user", - "server": {"username": "ubuntu", "password": "ubuntu.password"}, - "databases": [ - {"host": "www.db1.com", "username": "admin", "password": "db1pwd"}, - {"host": "www.db2.com", "username": "admin", "password": "db2pwd"}, - ], - "password": "test.password", - }, - "prod": {"password": "prod.password"}, - } - - data1 = { - "tags": [ - {"key1": "value1"}, - ] - } - data2 = { - "tags": [ - {"key2": "value2"}, - {"key2": "value2"}, - ] - } - with pytest.raises(ValueError): - merge_key_value(data1, data2) - - with pytest.raises(ValueError): - merge_key_value({"values": [1, 2]}, {"values": [2, 3]}) - - with pytest.raises(TypeError): - merge_key_value({"value": 1}, {"value": 2}) - - dir_here = Path(__file__).absolute().parent path_config = str(dir_here.joinpath("config.json")) path_secret_config = str(dir_here.joinpath("secret_config.json"))