From 952ace83e0c81e48881cc554c46b00a590221aec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Tue, 30 Apr 2024 14:06:43 -0600 Subject: [PATCH] test: Add more tests for `SelectService` and the `select` command --- src/meltano/core/elt_context.py | 2 +- tests/fixtures/core.py | 7 +- tests/meltano/cli/test_select.py | 43 +++++++++ tests/meltano/core/test_select_service.py | 105 ++++++++++++++++++++++ 4 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 tests/meltano/core/test_select_service.py diff --git a/src/meltano/core/elt_context.py b/src/meltano/core/elt_context.py index 87a5b5093d..189f68a8f5 100644 --- a/src/meltano/core/elt_context.py +++ b/src/meltano/core/elt_context.py @@ -85,7 +85,7 @@ class ELTContext: # noqa: WPS230 def __init__( self, - project, + project: Project, job: Job | None = None, session=None, extractor: PluginContext | None = None, diff --git a/tests/fixtures/core.py b/tests/fixtures/core.py index 2692869a70..a75f83efeb 100644 --- a/tests/fixtures/core.py +++ b/tests/fixtures/core.py @@ -1067,7 +1067,12 @@ def discovery(): # noqa: WPS213 "kind": "object", "value": {"nested": "from_default"}, }, - {"name": "hidden", "kind": "hidden", "value": 42}, + { + "name": "hidden", + "kind": "integer", + "value": 42, + "hidden": True, + }, {"name": "boolean", "kind": "boolean"}, {"name": "auth.username"}, {"name": "auth.password", "sensitive": True}, diff --git a/tests/meltano/cli/test_select.py b/tests/meltano/cli/test_select.py index 92050ffce5..0ba8d30b95 100644 --- a/tests/meltano/cli/test_select.py +++ b/tests/meltano/cli/test_select.py @@ -6,6 +6,12 @@ from asserts import assert_cli_runner from meltano.cli import cli +from meltano.core.plugin.singer.catalog import ( + ListSelectedExecutor, + SelectedNode, + SelectionType, +) +from meltano.core.select_service import SelectService class TestCliSelect: @@ -33,3 +39,40 @@ def test_update_select_pattern(self, cli_runner, tap): assert_cli_runner(result) json_config = json.loads(result.stdout) assert "mock.*" not in json_config["_select"] + + @pytest.mark.usefixtures("project") + def test_select_list(self, cli_runner, tap, monkeypatch: pytest.MonkeyPatch): + async def mock_list_all(*args, **kwargs): # noqa: ARG001 + result = ListSelectedExecutor() + result.streams = { + SelectedNode(key="users", selection=SelectionType.SELECTED) + } + result.properties = { + "users": { + SelectedNode(key="id", selection=SelectionType.SELECTED), + SelectedNode(key="name", selection=SelectionType.EXCLUDED), + } + } + return result + + monkeypatch.setattr( + SelectService, + "list_all", + mock_list_all, + ) + + # list selection + result = cli_runner.invoke( + cli, + [ + "--no-environment", + "select", + tap.name, + "--list", + "--all", + ], + ) + assert_cli_runner(result) + + assert "[SelectionType.SELECTED] users.id" in result.stdout + assert "[SelectionType.EXCLUDED] users.name" in result.stdout diff --git a/tests/meltano/core/test_select_service.py b/tests/meltano/core/test_select_service.py new file mode 100644 index 0000000000..3462b0bfb3 --- /dev/null +++ b/tests/meltano/core/test_select_service.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import json +import typing as t +from collections import OrderedDict + +import pytest + +from meltano.core.plugin.singer.catalog import SelectedNode, SelectionType +from meltano.core.plugin.singer.tap import SingerTap +from meltano.core.select_service import SelectService + +if t.TYPE_CHECKING: + from sqlalchemy.orm.session import Session + + from meltano.core.project import Project + + +@pytest.mark.asyncio() +@pytest.mark.usefixtures("tap") +async def test_select_service_list_all( + project: Project, + session: Session, + monkeypatch: pytest.MonkeyPatch, +): + catalog = { + "streams": [ + { + "stream": "users", + "tap_stream_id": "users", + "metadata": [ + {"breadcrumb": [], "metadata": {"selected": True}}, + ], + "schema": { + "properties": { + "id": {"type": "integer"}, + } + }, + } + ] + } + extractor = "tap-mock" + service = SelectService(project, extractor) + + async def mock_run_discovery(tap, plugin_invoker, catalog_path): # noqa: ARG001 + with catalog_path.open("w") as catalog_file: + json.dump(catalog, catalog_file) + + # Mock tap's run_discovery method + monkeypatch.setattr( + SingerTap, + "run_discovery", + mock_run_discovery, + ) + + list_all = await service.list_all(session, refresh=False) + assert list_all.streams == { + SelectedNode( + key="users", + selection=SelectionType.SELECTED, + ) + } + assert list_all.properties == OrderedDict( + { + "users": { + SelectedNode( + key="id", + selection=SelectionType.AUTOMATIC, + ), + } + } + ) + + # Update the catalog to include a new property + catalog["streams"][0]["schema"]["properties"]["name"] = {"type": "string"} # noqa: WPS219 + + # Without refreshing the catalog, the new property should not be included + list_all = await service.list_all(session, refresh=False) + assert list_all.properties == OrderedDict( + { + "users": { + SelectedNode( + key="id", + selection=SelectionType.AUTOMATIC, + ), + } + } + ) + + # Refreshing the catalog should include the new property + list_all = await service.list_all(session, refresh=True) + assert list_all.properties == OrderedDict( + { + "users": { + SelectedNode( + key="id", + selection=SelectionType.AUTOMATIC, + ), + SelectedNode( + key="name", + selection=SelectionType.AUTOMATIC, + ), + } + } + )