From 8eb94fec0f4d164771ed4f28a7d96b6e47bcbb16 Mon Sep 17 00:00:00 2001 From: Joao Mario Lago Date: Thu, 23 May 2024 11:32:07 -0300 Subject: [PATCH] common: settings: Add pydantic POC settings tests --- .../settings/tests/test_manager_pydantic.py | 178 +++++++++++++++ .../settings/tests/test_settings_pydantic.py | 215 ++++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100644 core/libs/commonwealth/commonwealth/settings/tests/test_manager_pydantic.py create mode 100644 core/libs/commonwealth/commonwealth/settings/tests/test_settings_pydantic.py diff --git a/core/libs/commonwealth/commonwealth/settings/tests/test_manager_pydantic.py b/core/libs/commonwealth/commonwealth/settings/tests/test_manager_pydantic.py new file mode 100644 index 0000000000..ee0a032219 --- /dev/null +++ b/core/libs/commonwealth/commonwealth/settings/tests/test_manager_pydantic.py @@ -0,0 +1,178 @@ +import os +import pathlib +import tempfile +from typing import Any, Dict + +from .. import manager_pydantic, settings_pydantic + + +class SettingsV1(settings_pydantic.PydanticSettings): + VERSION: int = 1 + version_1_variable: int = 42 + + def migrate(self, data: Dict[str, Any]) -> None: + if data["VERSION"] == self.VERSION: + return + + if data["VERSION"] < self.VERSION: + super().migrate(data) + + data["VERSION"] = self.VERSION + data["version_1_variable"] = self.version_1_variable + + +class SettingsV2(SettingsV1): + VERSION: int = 2 + version_2_variable: int = 66 + + def migrate(self, data: Dict[str, Any]) -> None: + if data["VERSION"] == self.VERSION: + return + + if data["VERSION"] < self.VERSION: + SettingsV1().migrate(data) + + data["VERSION"] = self.VERSION + data["version_2_variable"] = self.version_2_variable + + +class SettingsV3(SettingsV2): + VERSION: int = 3 + version_3_variable: int = 99 + + def migrate(self, data: Dict[str, Any]) -> None: + if data["VERSION"] == self.VERSION: + return + + if data["VERSION"] < self.VERSION: + SettingsV2().migrate(data) + + data["VERSION"] = self.VERSION + data["version_3_variable"] = self.version_3_variable + + +class SettingsV12(SettingsV3): + VERSION: int = 12 + version_12_variable: int = 1992 + + def migrate(self, data: Dict[str, Any]) -> None: + if data["VERSION"] == self.VERSION: + return + + if data["VERSION"] < self.VERSION: + SettingsV3().migrate(data) + + data["VERSION"] = self.VERSION + data["version_12_variable"] = self.version_12_variable + + +def test_basic_settings_save_load() -> None: + temporary_folder = tempfile.mkdtemp() + config_path = pathlib.Path(temporary_folder) + + # Test v1 save + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV1, config_path) + settings_manager.settings.version_1_variable = 2022 + settings_manager.save() + + assert settings_manager.settings.version_1_variable == 2022 + + # Test v1 load + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV1, config_path) + + assert settings_manager.settings.version_1_variable == 2022 + + # Test v2 load/save with migration from v1 + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV2, config_path) + + assert settings_manager.settings.version_1_variable == 2022 + assert settings_manager.settings.version_2_variable == 66 + + settings_manager.settings.version_2_variable = 123 + settings_manager.save() + + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV2, config_path) + + assert settings_manager.settings.version_1_variable == 2022 + assert settings_manager.settings.version_2_variable == 123 + + # Test v3 load/save with migration from v2 + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV3, config_path) + + assert settings_manager.settings.version_1_variable == 2022 + assert settings_manager.settings.version_2_variable == 123 + assert settings_manager.settings.version_3_variable == 99 + + settings_manager.settings.version_3_variable = 222 + settings_manager.save() + + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV3, config_path) + + assert settings_manager.settings.version_1_variable == 2022 + assert settings_manager.settings.version_2_variable == 123 + assert settings_manager.settings.version_3_variable == 222 + + # Test v12 load/save with migration from v3 + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV12, config_path) + + assert settings_manager.settings.version_1_variable == 2022 + assert settings_manager.settings.version_2_variable == 123 + assert settings_manager.settings.version_3_variable == 222 + assert settings_manager.settings.version_12_variable == 1992 + + settings_manager.settings.version_12_variable = 14 + settings_manager.save() + + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV12, config_path) + + assert settings_manager.settings.version_1_variable == 2022 + assert settings_manager.settings.version_2_variable == 123 + assert settings_manager.settings.version_3_variable == 222 + assert settings_manager.settings.version_12_variable == 14 + + assert len(os.listdir(config_path.joinpath("managertest"))) == 4 + + +def test_fallback_settings_save_load() -> None: + temporary_folder = tempfile.mkdtemp() + config_path = pathlib.Path(temporary_folder) + + # Test v12 with downgrade to v3 + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV12, config_path) + + assert settings_manager.settings.version_1_variable == SettingsV1().version_1_variable + assert settings_manager.settings.version_2_variable == SettingsV2().version_2_variable + assert settings_manager.settings.version_3_variable == SettingsV3().version_3_variable + assert settings_manager.settings.version_12_variable == SettingsV12().version_12_variable + + settings_manager.settings.version_1_variable = 1 + settings_manager.settings.version_2_variable = 2 + settings_manager.settings.version_3_variable = 3 + settings_manager.settings.version_12_variable = 12 + settings_manager.save() + + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV3, config_path) + + assert settings_manager.settings.version_1_variable == SettingsV1().version_1_variable + assert settings_manager.settings.version_2_variable == SettingsV2().version_2_variable + assert settings_manager.settings.version_3_variable == SettingsV3().version_3_variable + + settings_manager.settings.version_1_variable = 10 + settings_manager.settings.version_2_variable = 20 + settings_manager.settings.version_3_variable = 30 + settings_manager.save() + + # Check if v3 loads previous v3 configuration + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV3, config_path) + + assert settings_manager.settings.version_1_variable == 10 + assert settings_manager.settings.version_2_variable == 20 + assert settings_manager.settings.version_3_variable == 30 + + # Check if v12 loads previous v12 configuration without v3 migration + settings_manager = manager_pydantic.Manager("ManagerTest", SettingsV12, config_path) + + assert settings_manager.settings.version_1_variable == 1 + assert settings_manager.settings.version_2_variable == 2 + assert settings_manager.settings.version_3_variable == 3 + assert settings_manager.settings.version_12_variable == 12 diff --git a/core/libs/commonwealth/commonwealth/settings/tests/test_settings_pydantic.py b/core/libs/commonwealth/commonwealth/settings/tests/test_settings_pydantic.py new file mode 100644 index 0000000000..9807e331b3 --- /dev/null +++ b/core/libs/commonwealth/commonwealth/settings/tests/test_settings_pydantic.py @@ -0,0 +1,215 @@ +import pathlib +import tempfile +from typing import Any, Dict, List + +from pydantic import BaseModel + +from .. import settings_pydantic + + +class JsonExample(BaseModel): + name: str = "" + + +class Animal(BaseModel): + name: str = "" + animal_type: str = "" + parts: List[str] = [] + animal_json: List[JsonExample] = [] + + +class SettingsV1(settings_pydantic.PydanticSettings): + VERSION: int = 1 + + animal: Animal = Animal( + name="bilica", + animal_type="dog", + parts=["finger", "eyes"], + animal_json=[JsonExample.parse_obj({"name": "Json!"})], + ) + first_variable: int = 42 + + def migrate(self, data: Dict[str, Any]) -> None: + if data["VERSION"] == self.VERSION: + return + + if data["VERSION"] < self.VERSION: + super().migrate(data) + + data["VERSION"] = self.VERSION + data["animal"] = self.animal + data["first_variable"] = self.first_variable + + +class SettingsV1Expanded(SettingsV1): + new_variable: int = 1992 + + +class SettingsV2(settings_pydantic.PydanticSettings): + VERSION: int = 2 + first_variable: int = 66 + new_animal: Animal = Animal( + name="bilica", + animal_type="dog", + ) + + def migrate(self, data: Dict[str, Any]) -> None: + if data["VERSION"] == self.VERSION: + return + + if data["VERSION"] < self.VERSION: + SettingsV1().migrate(data) + + data["VERSION"] = self.VERSION + data["first_variable"] = self.first_variable + + # Update variable name + data["new_animal"] = data["animal"] + data.pop("animal") + + +class SettingsV3(SettingsV2): + VERSION: int = 3 + + def migrate(self, data: Dict[str, Any]) -> None: + if data["VERSION"] == self.VERSION: + return + + if data["VERSION"] < self.VERSION: + super().migrate(data) + + data["VERSION"] = self.VERSION + + +class SettingsV4(SettingsV3): + VERSION: int = 4 + + def migrate(self, data: Dict[str, Any]) -> None: + if data["VERSION"] == self.VERSION: + return + + if data["VERSION"] < self.VERSION: + super().migrate(data) + + data["VERSION"] = self.VERSION + + +def test_basic_settings_save_load() -> None: + # Check basic access + settings_v1 = SettingsV1() + assert settings_v1.VERSION == 1 + assert settings_v1.first_variable == 42 + assert settings_v1.animal.name == "bilica" + assert settings_v1.animal.animal_type == "dog" + assert settings_v1.animal.parts == ["finger", "eyes"] + assert settings_v1.animal.animal_json[0].name == "Json!" + + # pylint: disable=consider-using-with + temporary_file = tempfile.NamedTemporaryFile().name + file_path = pathlib.Path(temporary_file) + + # Check basic save and load + settings_v1.first_variable = 66 + settings_v1.save(file_path) + + settings_v1_new = SettingsV1() + settings_v1_new.load(file_path) + assert settings_v1.first_variable == settings_v1_new.first_variable + + # Check for reset + settings_v1_new.reset() + settings_v1.save(file_path) + + settings_v1_new = SettingsV1() + settings_v1_new.load(file_path) + assert settings_v1.first_variable == 66 + + +def test_nested_settings_save_load() -> None: + # Check basic access + settings_v1 = SettingsV1() + assert settings_v1.animal.name == SettingsV1().animal.name + assert settings_v1.animal.animal_type == SettingsV1().animal.animal_type + + # pylint: disable=consider-using-with + temporary_file = tempfile.NamedTemporaryFile() + file_path = pathlib.Path(temporary_file.name) + + # Check basic save and load + settings_v1.first_variable = 66 + settings_v1.animal.name = "pingu" + settings_v1.animal.animal_type = "penguin" + + assert settings_v1.first_variable == 66 + assert settings_v1.animal.name == "pingu" + assert settings_v1.animal.animal_type == "penguin" + + settings_v1.save(file_path) + settings_v1_new = SettingsV1() + settings_v1_new.load(file_path) + + assert settings_v1.first_variable == settings_v1_new.first_variable + assert settings_v1.animal.name == settings_v1_new.animal.name + assert settings_v1.animal.animal_type == settings_v1_new.animal.animal_type + + +def test_simple_migration_settings_save_load() -> None: + settings_v1 = SettingsV1() + + # pylint: disable=consider-using-with + temporary_file = tempfile.NamedTemporaryFile() + file_path = pathlib.Path(temporary_file.name) + + settings_v1.first_variable = 66 + settings_v1.animal.name = "pingu" + settings_v1.animal.animal_type = "penguin" + + settings_v1.save(file_path) + settings_v1_new = SettingsV1() + settings_v1_new.load(file_path) + + # Check if migration works + settings_v2 = SettingsV2() + settings_v2.load(file_path) + settings_v2.save(file_path) + + assert settings_v1.first_variable == settings_v2.first_variable + assert settings_v1.animal.name == settings_v2.new_animal.name + assert settings_v1.animal.animal_type == settings_v2.new_animal.animal_type + + settings_v3 = SettingsV3() + settings_v3.load(file_path) + settings_v3.save(file_path) + + settings_v4 = SettingsV4() + settings_v4.load(file_path) + settings_v4.save(file_path) + + assert settings_v2.first_variable == settings_v4.first_variable + assert settings_v2.new_animal.name == settings_v4.new_animal.name + assert settings_v2.new_animal.animal_type == settings_v4.new_animal.animal_type + assert settings_v2.new_animal.parts[0] == settings_v4.new_animal.parts[0] + assert settings_v2.new_animal.parts[1] == settings_v4.new_animal.parts[1] + assert settings_v2.new_animal.animal_json[0].name == settings_v4.new_animal.animal_json[0].name + + +def test_simple_settings_expanded_save_load() -> None: + settings_v1 = SettingsV1() + + # pylint: disable=consider-using-with + temporary_file = tempfile.NamedTemporaryFile() + file_path = pathlib.Path(temporary_file.name) + + settings_v1.first_variable = 66 + settings_v1.animal.name = "pingu" + settings_v1.animal.animal_type = "penguin" + + # Load expanded settings with older settings structure + settings_v1.save(file_path) + settings_v1_expanded = SettingsV1Expanded() + settings_v1_expanded.load(file_path) + + assert settings_v1.first_variable == settings_v1_expanded.first_variable + assert settings_v1.animal.name == settings_v1_expanded.animal.name + assert settings_v1.animal.animal_type == settings_v1_expanded.animal.animal_type + assert settings_v1_expanded.new_variable == 1992