From 00a5a674a725300a3c0722b602aae0867815d7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 16 Nov 2023 19:03:48 -0600 Subject: [PATCH] feat: Support state backend core setting extensions --- poetry.lock | 51 ++++++++++- pyproject.toml | 1 + src/meltano/core/behavior/pluggable.py | 54 +++++++++++ src/meltano/core/config_service.py | 12 ++- src/meltano/core/state_store/__init__.py | 90 ++++++++++++++++--- tests/fixtures/custom_pluggable.py | 5 ++ tests/fixtures/state_backends.py | 25 ++++++ tests/meltano/core/behavior/test_pluggable.py | 43 +++++++++ .../core/state_store/test_state_store.py | 50 ++++++++++- 9 files changed, 311 insertions(+), 20 deletions(-) create mode 100644 src/meltano/core/behavior/pluggable.py create mode 100644 tests/fixtures/custom_pluggable.py create mode 100644 tests/fixtures/state_backends.py create mode 100644 tests/meltano/core/behavior/test_pluggable.py diff --git a/poetry.lock b/poetry.lock index 414eea2e99..349c94f789 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2143,6 +2143,55 @@ files = [ {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, ] [[package]] @@ -4272,4 +4321,4 @@ s3 = ["boto3"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.13" -content-hash = "3d73f723abc94d761d98e5a99c713dca7cc3a075e99b3c7edbc58bea46f334ee" +content-hash = "822b5acade88231b6d266316ae6d9d50966a2b08a1a332df9085786c2f7aebba" diff --git a/pyproject.toml b/pyproject.toml index 03fec0bed0..24c8929c74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ croniter = "^2.0.1" fasteners = "^0.19" flatten-dict = "^0" google-cloud-storage = {version = ">=1.31.0", optional = true} +importlib-metadata = { version = ">=5", python = "<3.12" } importlib-resources = "^6.1.1" jinja2 = "^3.1.3" jsonschema = "^4.21" diff --git a/src/meltano/core/behavior/pluggable.py b/src/meltano/core/behavior/pluggable.py new file mode 100644 index 0000000000..b70e0ad65a --- /dev/null +++ b/src/meltano/core/behavior/pluggable.py @@ -0,0 +1,54 @@ +"""Make a feature pluggable with packaging plugins.""" + +from __future__ import annotations + +import sys +import typing as t +from functools import cached_property + +if sys.version_info >= (3, 12): + from importlib.metadata import EntryPoints, entry_points +else: + from importlib_metadata import EntryPoints, entry_points + +T = t.TypeVar("T") + + +class Pluggable(t.Generic[T]): + """Make a feature pluggable with packaging plugins.""" + + def __init__(self, entry_point_group: str): + """Create a new pluggable feature. + + Args: + entry_point_group: The entry point group for the feature. + """ + self.entry_point_group = entry_point_group + + @cached_property + def meltano_plugins(self) -> EntryPoints: + """List available plugins. + + Returns: + List of available plugins. + """ + return entry_points(group=self.entry_point_group) + + def get_all_meltano_plugins(self) -> dict[str, T]: + """Get all plugins. + + Returns: + All plugins. + """ + return {ep.name: ep.load() for ep in self.meltano_plugins} + + def get_meltano_plugin(self, name: str) -> T: + """Get a plugin by name. + + Args: + name: The name of the plugin. + + Returns: + The plugged object. + """ + return self.meltano_plugins[name].load() diff --git a/src/meltano/core/config_service.py b/src/meltano/core/config_service.py index 6d0624552a..75fdc27dc2 100644 --- a/src/meltano/core/config_service.py +++ b/src/meltano/core/config_service.py @@ -11,6 +11,7 @@ import yaml from meltano.core import bundle +from meltano.core.behavior.pluggable import Pluggable from meltano.core.setting_definition import SettingDefinition if t.TYPE_CHECKING: @@ -20,9 +21,11 @@ logger = logging.getLogger(__name__) -class ConfigService: +class ConfigService(Pluggable[SettingDefinition]): # noqa: WPS214 """Service to manage meltano.yml.""" + settings_plugins: Pluggable[SettingDefinition] = Pluggable("meltano.settings") + def __init__(self, project: Project): """Create a new project configuration service. @@ -40,7 +43,12 @@ def settings(self) -> list[SettingDefinition]: """ with open(str(bundle.root / "settings.yml")) as settings_yaml: settings_yaml_content = yaml.safe_load(settings_yaml) - return [SettingDefinition.parse(x) for x in settings_yaml_content["settings"]] + + builtin = [ + SettingDefinition.parse(x) for x in settings_yaml_content["settings"] + ] + plugged = list(self.settings_plugins.get_all_meltano_plugins().values()) + return builtin + plugged @cached_property def current_meltano_yml(self) -> MeltanoFile: diff --git a/src/meltano/core/state_store/__init__.py b/src/meltano/core/state_store/__init__.py index f0799dc9ad..96b6c32d35 100644 --- a/src/meltano/core/state_store/__init__.py +++ b/src/meltano/core/state_store/__init__.py @@ -2,12 +2,17 @@ from __future__ import annotations import platform +import sys import typing as t from enum import Enum from urllib.parse import urlparse +from structlog.stdlib import get_logger + +from meltano.core.behavior.pluggable import Pluggable from meltano.core.db import project_engine from meltano.core.state_store.azure import AZStorageStateStoreManager +from meltano.core.state_store.base import StateStoreManager from meltano.core.state_store.db import DBStateStoreManager from meltano.core.state_store.filesystem import ( LocalFilesystemStateStoreManager, @@ -20,10 +25,28 @@ from sqlalchemy.orm import Session from meltano.core.project_settings_service import ProjectSettingsService - from meltano.core.state_store.base import StateStoreManager + +if sys.version_info >= (3, 12): + from importlib.metadata import EntryPoints, entry_points +else: + from importlib_metadata import EntryPoints, entry_points + +__all__ = [ + "AZStorageStateStoreManager", + "BuiltinStateBackendEnum", + "DBStateStoreManager", + "GCSStateStoreManager", + "LocalFilesystemStateStoreManager", + "S3StateStoreManager", + "StateBackend", + "StateStoreManager", + "state_store_manager_from_project_settings", +] + +logger = get_logger(__name__) -class StateBackend(str, Enum): +class BuiltinStateBackendEnum(str, Enum): """State backend.""" SYSTEMDB = "systemdb" @@ -35,17 +58,35 @@ class StateBackend(str, Enum): S3 = "s3" GCS = "gs" + +class StateBackend: + """State backend.""" + + state_backend_plugins: Pluggable[type[StateStoreManager]] = Pluggable( + "meltano.state_backends", + ) + + def __init__(self, scheme: str) -> None: + """Create a new StateBackend. + + Args: + scheme: The scheme of the StateBackend. + """ + self.scheme = scheme + @classmethod - def backends(cls) -> list[StateBackend]: + def backends(cls) -> list[str]: """List available state backends. Returns: List of available state backends. """ - return list(cls) + return list(BuiltinStateBackendEnum) + [ + ep.name for ep in cls.state_backend_plugins.meltano_plugins + ] @property - def _managers( + def _builtin_managers( self, ) -> t.Mapping[str, type[StateStoreManager]]: """Get mapping of StateBackend to associated StateStoreManager. @@ -54,11 +95,11 @@ def _managers( Mapping of StateBackend to associated StateStoreManager. """ return { - self.SYSTEMDB: DBStateStoreManager, - self.LOCAL_FILESYSTEM: LocalFilesystemStateStoreManager, - self.S3: S3StateStoreManager, - self.AZURE: AZStorageStateStoreManager, - self.GCS: GCSStateStoreManager, + BuiltinStateBackendEnum.SYSTEMDB: DBStateStoreManager, + BuiltinStateBackendEnum.LOCAL_FILESYSTEM: LocalFilesystemStateStoreManager, + BuiltinStateBackendEnum.S3: S3StateStoreManager, + BuiltinStateBackendEnum.AZURE: AZStorageStateStoreManager, + BuiltinStateBackendEnum.GCS: GCSStateStoreManager, } @property @@ -69,8 +110,28 @@ def manager( Returns: The StateStoreManager associated with this StateBackend. + + Raises: + ValueError: If no state backend is found for the scheme. """ - return self._managers[self] + try: + return self._builtin_managers[self.scheme] + except KeyError: + logger.info( + "No builtin state backend found for scheme '%s'", # noqa: WPS323 + self.scheme, + ) + + try: + return self.state_backend_plugins.get_meltano_plugin(self.scheme) + except KeyError: + logger.info( + "No state backend plugin found for scheme '%s'", # noqa: WPS323 + self.scheme, + ) + + # TODO: This should be a Meltano exception + raise ValueError(f"No state backend found for scheme '{self.scheme}'") def state_store_manager_from_project_settings( # noqa: WPS210 @@ -88,7 +149,7 @@ def state_store_manager_from_project_settings( # noqa: WPS210 """ state_backend_uri: str = settings_service.get("state_backend.uri") parsed = urlparse(state_backend_uri) - if state_backend_uri == StateBackend.SYSTEMDB: + if state_backend_uri == BuiltinStateBackendEnum.SYSTEMDB: return DBStateStoreManager( session=session or project_engine(settings_service.project)[1](), ) @@ -107,7 +168,10 @@ def state_store_manager_from_project_settings( # noqa: WPS210 ) settings = (setting_def.name for setting_def in setting_defs) backend = StateBackend(scheme).manager - if scheme == StateBackend.LOCAL_FILESYSTEM and "Windows" in platform.system(): + if ( + scheme == BuiltinStateBackendEnum.LOCAL_FILESYSTEM + and "Windows" in platform.system() + ): backend = WindowsFilesystemStateStoreManager kwargs = {name.split(".")[-1]: settings_service.get(name) for name in settings} return backend(**kwargs) diff --git a/tests/fixtures/custom_pluggable.py b/tests/fixtures/custom_pluggable.py new file mode 100644 index 0000000000..625b464038 --- /dev/null +++ b/tests/fixtures/custom_pluggable.py @@ -0,0 +1,5 @@ +from __future__ import annotations + + +class MyPlugin: + """Dummy plugin.""" diff --git a/tests/fixtures/state_backends.py b/tests/fixtures/state_backends.py new file mode 100644 index 0000000000..04e74a2996 --- /dev/null +++ b/tests/fixtures/state_backends.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from meltano.core.state_store import StateStoreManager + + +class DummyStateStoreManager(StateStoreManager): + label: str = "Dummy state store manager" + + def __init__(self, **kwargs): + ... # noqa: WPS428 + + def set(self, state): + ... # noqa: WPS428 + + def get(self, state_id): + ... # noqa: WPS428 + + def clear(self, state_id): + ... # noqa: WPS428 + + def get_state_ids(self, _pattern=None): + return () # pragma: no cover + + def acquire_lock(self, state_id): + ... # noqa: WPS428 diff --git a/tests/meltano/core/behavior/test_pluggable.py b/tests/meltano/core/behavior/test_pluggable.py new file mode 100644 index 0000000000..0de8337eb5 --- /dev/null +++ b/tests/meltano/core/behavior/test_pluggable.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import sys +import typing as t + +from fixtures.custom_pluggable import MyPlugin +from meltano.core.behavior.pluggable import Pluggable + +if sys.version_info >= (3, 12): + from importlib.metadata import EntryPoint, EntryPoints +else: + from importlib_metadata import EntryPoint, EntryPoints + +if t.TYPE_CHECKING: + import pytest + + +def test_entry_point_group(): + """Test that the entry point group is set correctly.""" + pluggable = Pluggable("meltano.custom_feature") + + assert pluggable.entry_point_group == "meltano.custom_feature" + assert not pluggable.meltano_plugins + + +def test_entry_points(monkeypatch: pytest.MonkeyPatch): + """Test that the entry points are loaded correctly.""" + pluggable: Pluggable[object] = Pluggable("meltano.custom_feature") + + entry_points = EntryPoints( + ( + EntryPoint( + value="fixtures.custom_pluggable:MyPlugin", + name="custom", + group="meltano.custom_feature", + ), + ), + ) + + with monkeypatch.context() as m: + m.setattr(pluggable, "meltano_plugins", entry_points) + plugin = pluggable.get_meltano_plugin("custom") + assert plugin == MyPlugin diff --git a/tests/meltano/core/state_store/test_state_store.py b/tests/meltano/core/state_store/test_state_store.py index 38874e66ea..ebd3899d0d 100644 --- a/tests/meltano/core/state_store/test_state_store.py +++ b/tests/meltano/core/state_store/test_state_store.py @@ -1,6 +1,7 @@ from __future__ import annotations import shutil +import sys import typing as t import pytest @@ -10,9 +11,11 @@ SharedKeyCredentialPolicy, ) +from fixtures.state_backends import DummyStateStoreManager from meltano.core.error import MeltanoError -from meltano.core.state_store import ( +from meltano.core.state_store import ( # noqa: WPS235 AZStorageStateStoreManager, + BuiltinStateBackendEnum, DBStateStoreManager, GCSStateStoreManager, LocalFilesystemStateStoreManager, @@ -26,10 +29,42 @@ from meltano.core.project import Project +if sys.version_info >= (3, 12): + from importlib.metadata import EntryPoint, EntryPoints +else: + from importlib_metadata import EntryPoint, EntryPoints + + +def test_unknown_state_backend_scheme(project: Project): + project.settings.set(["state_backend", "uri"], "unknown://") + with pytest.raises(ValueError, match="No state backend found for scheme"): + state_store_manager_from_project_settings(project.settings) + + +def test_pluggable_state_backend(project: Project, monkeypatch: pytest.MonkeyPatch): + project.settings.set(["state_backend", "uri"], "custom://") + + entry_points = EntryPoints( + ( + EntryPoint( + value="fixtures.state_backends:DummyStateStoreManager", + name="custom", + group="meltano.state_backends", + ), + ), + ) + + with monkeypatch.context() as m: + m.setattr(StateBackend.state_backend_plugins, "meltano_plugins", entry_points) + assert "custom" in StateBackend.backends() + + state_store = state_store_manager_from_project_settings(project.settings) + assert isinstance(state_store, DummyStateStoreManager) + class TestSystemDBStateBackend: def test_manager_from_settings(self, project: Project): - project.settings.set(["state_backend", "uri"], StateBackend.SYSTEMDB) + project.settings.set(["state_backend", "uri"], BuiltinStateBackendEnum.SYSTEMDB) project.settings.set(["state_backend", "lock_timeout_seconds"], 10) db_state_store = state_store_manager_from_project_settings(project.settings) assert isinstance(db_state_store, DBStateStoreManager) @@ -44,9 +79,16 @@ def state_path(self, tmp_path: Path): finally: shutil.rmtree(path) - def test_manager_from_settings(self, project: Project, state_path: str): + def test_manager_from_settings( + self, + project: Project, + state_path: str, + ): + def get_state_store(): + return state_store_manager_from_project_settings(project.settings) + project.settings.set(["state_backend", "uri"], f"file://{state_path}") - file_state_store = state_store_manager_from_project_settings(project.settings) + file_state_store = get_state_store() assert isinstance(file_state_store, LocalFilesystemStateStoreManager) assert file_state_store.state_dir == state_path