Add tests and docs for environment variable resolution.

CHANGELOG.md

@@ -16,6 +16,7 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch
 - [#5923](https://github.com/meltano/meltano/pull/5923) Add support for jobs to schedules and improves general schedule cli UX
 
 ### Changes
+- [#3173](https://gitlab.com/meltano/meltano/-/issues/3173) Add tests and clarify docs for environment variable resolution.
 
 - [#3427](https://gitlab.com/meltano/meltano/-/issues/3427) Fully remove option for Explore/Dashboard UI in 2.0
 

docs/src/_guide/configuration.md

@@ -30,7 +30,7 @@ To determine the values of settings, Meltano will look in 4 main places (and one
 2. **Your [`meltano.yml` project file](/concepts/project#meltano-yml-project-file)**, under the plugin's `config` key.
    - Inside values, [environment variables can be referenced](#expansion-in-setting-values) as `$VAR` (as a single word) or `${VAR}` (inside a word).
    - Note that configuration for Meltano itself is stored at the root level of `meltano.yml`.
-   - You can use [Meltano Environments](/concepts/environments) to manage different configurations depending on your testing and deployment strategy.
+   - You can use [Meltano Environments](/concepts/environments) to manage different configurations depending on your testing and deployment strategy. If values for plugin settings are provided in both the top-level plugin configuration _and_ the environment-level plugin configuration, the value at the environment level will take precedence.
 3. **Your project's [**system database**](/concepts/project#system-database)**, which (among other things) stores configuration set using [`meltano config <plugin> set`](/reference/command-line-interface#config) or [the UI](/reference/ui) when the project is [deployed as read-only](/reference/settings#project-readonly).
    - Note that configuration for Meltano itself cannot be stored in the system database.
 4. _If the plugin [inherits from another plugin](/concepts/plugins#plugin-inheritance) in your project_: **The parent plugin's own configuration**
@@ -126,6 +126,15 @@ When Meltano invokes a plugin's executable as part of [`meltano elt`](/reference
 
 These can then be accessed from inside the plugin using the mechanism provided by the standard library, e.g. Python's [`os.environ`](https://docs.python.org/3/library/os.html#os.environ).
 
+Within a [Meltano environment](/concepts/environments) environment variables can be specified using the `env` key:
+```yml
+environments:
+  - name: dev
+    env:
+      AN_ENVIRONMENT_VARIABLE: dev
+```
+Any plugins run in that Meltano environment will then have the provided environment variables populated into the plugin's environment.
+
 ## Multiple plugin configurations
 
 Every [plugin in your project](/concepts/plugins#project-plugins) has its own configuration,

docs/src/_reference/command-line-interface.md

@@ -591,6 +591,20 @@ To execute plugins inside containers, use the `--containers` flag:
 meltano invoke --containers dbt:compile
 ```
 
+### Debugging plugin environment
+
+When debugging plugin configuration, it is often useful to view environment variables being provided to a plugin at runtime.
+This can be achieved using `--log-level=debug` but for readability and convenience, the `meltano invoke` command also supports printing individual environment variables to stdout at runtime:
+```bash
+# Print the runtime value PLUGIN_ENVIRONMENT_VARIABLE_1:
+meltano invoke --print-var <PLUGIN_ENVIRONMENT_VARIABLE_1> <PLUGIN_NAME>
+
+# The option supports printing multiple variables as well.
+
+# # Print the runtime values of both PLUGIN_ENVIRONMENT_VARIABLE_1 and PLUGIN_ENVIRONMENT_2 on separate lines:
+meltano invoke --print-var <PLUGIN_ENVIRONMENT_VARIABLE_1> --print-var <PLUGIN_ENVIRONMENT_VARIABLE_2> <PLUGIN_NAME>
+```
+
 ## `remove`
 
 `meltano remove` removes one or more [plugins](/concepts/plugins#project-plugins) of the same [type](/concepts/plugins#types) from your Meltano [project](/concepts/project).

src/meltano/cli/invoke.py

@@ -1,8 +1,9 @@
 """CLI command `meltano invoke`."""
 
+from __future__ import annotations
+
 import logging
 import sys
-from typing import Tuple
 
 import click
 from sqlalchemy.orm import sessionmaker
@@ -31,6 +32,11 @@
     context_settings={"ignore_unknown_options": True, "allow_interspersed_args": False},
     short_help="Invoke a plugin.",
 )
+@click.option(
+    "--print-var",
+    help="Print to stdout the values for the provided environment variables, as passed to the plugininvoker context. Useful for debugging.",
+    multiple=True,
+)
 @click.option(
     "--plugin-type", type=click.Choice(PluginType.cli_arguments()), default=None
 )
@@ -58,8 +64,9 @@ def invoke(
     dump: str,
     list_commands: bool,
     plugin_name: str,
-    plugin_args: Tuple[str, ...],
+    plugin_args: tuple[str, ...],
     containers: bool = False,
+    print_var: str | None = None,
 ):
     """
     Invoke a plugin's executable with specified arguments.
@@ -95,6 +102,7 @@ def invoke(
             dump,
             command_name,
             containers,
+            print_var=print_var,
         )
     )
     sys.exit(exit_code)
@@ -109,12 +117,18 @@ async def _invoke(
     dump: str,
     command_name: str,
     containers: bool,
+    print_var: list | None = None,
 ):
     if command_name is not None:
         command = invoker.find_command(command_name)
 
     try:
         async with invoker.prepared(session):
+            if print_var:
+                env = invoker.env()
+                for key in print_var:
+                    val = env.get(key)
+                    click.echo(f"{key}={val}")
             if dump:
                 await dump_file(invoker, dump)
                 exit_code = 0

src/meltano/core/settings_store.py

@@ -6,6 +6,8 @@
 from contextlib import contextmanager
 from copy import deepcopy
 from enum import Enum
+from functools import reduce
+from operator import eq
 from typing import TYPE_CHECKING, Any
 
 import dotenv
@@ -26,6 +28,27 @@
 logger = logging.getLogger(__name__)
 
 
+class ConflictingSettingValueException(Exception):
+    """Occurs when a setting has multiple conflicting values via aliases."""
+
+    def __init__(self, *setting_names):
+        """Instantiate the error.
+
+        Args:
+            setting_names: the name/aliases where conflicting values are set
+
+        """
+        super().__init__()
+
+    def __str__(self) -> str:
+        """Represent the error as a string.
+
+        Returns:
+            string representation of the error
+        """
+        return f"Conflicting values for setting found in: {self.setting_names}"
+
+
 class StoreNotSupportedError(Error):
     """Error raised when write actions are performed on a Store that is not writable."""
 
@@ -264,21 +287,28 @@ def get(
 
         Raises:
             StoreNotSupportedError: if setting_def not passed.
+            ConflictingSettingValueException: if multiple conflicting values for the same setting are provided.
 
         Returns:
             A tuple the got value and a dictionary containing metadata.
         """
         if not setting_def:
             raise StoreNotSupportedError
 
+        vals_with_metadata = []
         for env_var in self.setting_env_vars(setting_def):
             try:
                 value = env_var.get(self.env)
-                return value, {"env_var": env_var.key}
+                vals_with_metadata.append((value, {"env_var": env_var.key}))
             except KeyError:
                 pass
-
-        return None, {}
+        if len(vals_with_metadata) > 1 and not reduce(
+            eq, (val for val, _ in vals_with_metadata)
+        ):
+            raise ConflictingSettingValueException(
+                metadata["env_var"] for _, metadata in vals_with_metadata
+            )
+        return vals_with_metadata[0] if vals_with_metadata else (None, {})
 
     def setting_env_vars(self, *args, **kwargs) -> dict:
         """Return setting environment variables.
@@ -523,22 +553,30 @@ def get(
 
         Returns:
             A tuple the got value and a dictionary containing metadata.
+
+        Raises:
+            ConflictingSettingValueException: if multiple conflicting values for the same setting are provided.
         """
         keys = [name]
         if setting_def:
             keys = [setting_def.name, *setting_def.aliases]
 
         flat_config = self.flat_config
-
+        vals_with_metadata = []
         for key in keys:
             try:
                 value = flat_config[key]
                 self.log(f"Read key '{key}' from `meltano.yml`: {value!r}")
-                return value, {"key": key, "expandable": True}
+                vals_with_metadata.append((value, {"key": key, "expandable": True}))
             except KeyError:
                 pass
-
-        return None, {}
+        if len(vals_with_metadata) > 1 and not reduce(
+            eq, (val for val, _ in vals_with_metadata)
+        ):
+            raise ConflictingSettingValueException(
+                metadata["key"] for _, metadata in vals_with_metadata
+            )
+        return vals_with_metadata[0] if vals_with_metadata else (None, {})
 
     def set(
         self,

tests/fixtures/core.py

@@ -79,6 +79,12 @@ def discovery():  # noqa: WPS213
                         },
                         {"name": "auth.username"},
                         {"name": "auth.password", "kind": "password"},
+                        {
+                            "name": "aliased",
+                            "kind": "string",
+                            "aliases": ["aliased_1", "aliased_2", "aliased_3"],
+                            "env_aliases": ["TAP_MOCK_ALIASED_ALIASED_1"],
+                        },
                     ],
                     "commands": {
                         "cmd": {
@@ -130,7 +136,12 @@ def discovery():  # noqa: WPS213
             "name": "target-mock",
             "namespace": "mock",
             "pip_url": "target-mock",
-            "settings": [{"name": "schema", "env": "MOCKED_SCHEMA"}],
+            "settings": [
+                {
+                    "name": "schema",
+                    "env": "MOCKED_SCHEMA",
+                }
+            ],
         }
     )
 

tests/meltano/core/plugin/test_plugin_settings.py

@@ -14,6 +14,7 @@
 from meltano.core.project import Project
 from meltano.core.project_plugins_service import PluginAlreadyAddedException
 from meltano.core.setting import Setting
+from meltano.core.settings_store import ConflictingSettingValueException
 
 
 def test_create(session):
@@ -172,14 +173,15 @@ def test_get_with_source_casting(self, session, subject, env_var, monkeypatch):
         )
 
         # Regular alias
+        monkeypatch.delenv("TAP_MOCK_DISABLED")
         monkeypatch.setenv("TAP_MOCK_ENABLED", "on")
-
         assert subject.get_with_source("boolean", session=session) == (
             True,
             SettingValueStore.ENV,
         )
 
         # Preferred env var
+        monkeypatch.delenv("TAP_MOCK_ENABLED")
         monkeypatch.setenv(env_var(subject, "boolean"), "0")
 
         assert subject.get_with_source("boolean", session=session) == (
@@ -351,14 +353,13 @@ def assert_env_value(value, env_var):
         assert_env_value("namespace_prefix", "MOCK_SCHEMA")
 
         # Name prefix
+        dotenv.unset_key(project.dotenv, "MOCK_SCHEMA")
         dotenv.set_key(project.dotenv, "TARGET_MOCK_SCHEMA", "name_prefix")
         assert_env_value("name_prefix", "TARGET_MOCK_SCHEMA")
 
         config = subject.as_env(session=session)
         subject.reset(store=SettingValueStore.DOTENV)
 
-        assert config["TARGET_MOCK_SCHEMA"] == "name_prefix"  # Name prefix
-        assert config["MOCK_SCHEMA"] == "name_prefix"  # Namespace prefix
         assert (
             config["MELTANO_LOAD_SCHEMA"] == "name_prefix"
         )  # Generic prefix, read-only
@@ -482,6 +483,8 @@ def test_store_dotenv(self, subject, project, tap):
         )
 
         dotenv.set_key(project.dotenv, "TAP_MOCK_DISABLED", "true")
+        assert subject.get_with_source("boolean") == (False, SettingValueStore.DOTENV)
+        dotenv.unset_key(project.dotenv, "TAP_MOCK_DISABLED")
         dotenv.set_key(project.dotenv, "TAP_MOCK_ENABLED", "false")
         assert subject.get_with_source("boolean") == (False, SettingValueStore.DOTENV)
 
@@ -875,3 +878,17 @@ def test_extra_object(
             {"var": "from_env"},
             SettingValueStore.ENV,
         )
+
+    def test_find_setting_raises_with_multiple(
+        self, tap, plugin_settings_service_factory, monkeypatch
+    ):
+        subject = plugin_settings_service_factory(tap)
+        monkeypatch.setenv("TAP_MOCK_ALIASED", "value_0")
+        monkeypatch.setenv("TAP_MOCK_ALIASED_ALIASED_1", "value_1")
+        with pytest.raises(ConflictingSettingValueException):
+            subject.get("aliased")
+
+    def test_find_setting_aliases(self, tap, plugin_settings_service_factory):
+        subject = plugin_settings_service_factory(tap)
+        subject.set("aliased_3", "value_3")
+        assert subject.get("aliased") == "value_3"