refactor: Use importlib.resources to read distribution files

src/meltano/core/bundle/__init__.py

@@ -2,6 +2,6 @@
 
 from __future__ import annotations
 
-from pathlib import Path
+from meltano.core.utils.compat import importlib_resources
 
-root = Path(__file__).resolve().parent
+root = importlib_resources.files(__package__)

src/meltano/core/config_service.py

@@ -38,7 +38,7 @@ def settings(self) -> list[SettingDefinition]:
         Returns:
             The project settings.
         """
-        with open(str(bundle.root / "settings.yml")) as settings_yaml:
+        with bundle.root.joinpath("settings.yml").open() as settings_yaml:
             settings_yaml_content = yaml.safe_load(settings_yaml)
         return [SettingDefinition.parse(x) for x in settings_yaml_content["settings"]]
 

src/meltano/core/manifest/manifest.py

@@ -11,14 +11,13 @@
 from contextlib import suppress
 from functools import cached_property, reduce
 from operator import getitem
-from pathlib import Path
 from tempfile import NamedTemporaryFile
 
 import flatten_dict  # type: ignore
 import structlog
 import yaml
 
-from meltano import __file__ as package_root_path
+from meltano import schemas
 from meltano.core.manifest.jsonschema import meltano_config_env_locations
 from meltano.core.plugin.settings_service import PluginSettingsService
 from meltano.core.plugin_lock_service import PluginLock
@@ -30,10 +29,12 @@
     expand_env_vars,
     get_no_color_flag,
 )
+from meltano.core.utils.compat import importlib_resources
 
 if t.TYPE_CHECKING:
     import sys
     from collections.abc import Iterable
+    from pathlib import Path
 
     from meltano.core.plugin.base import PluginType
     from meltano.core.plugin.project_plugin import ProjectPlugin
@@ -57,9 +58,7 @@
 logger = structlog.getLogger(__name__)
 
 JSON_LOCATION_PATTERN = re.compile(r"\.|(\[\])")
-MANIFEST_SCHEMA_PATH = (
-    Path(package_root_path).parent / "schemas" / "meltano.schema.json"
-)
+MANIFEST_SCHEMA_PATH = importlib_resources.files(schemas) / "meltano.schema.json"
 
 Trie: TypeAlias = t.Dict[str, "Trie"]
 PluginsByType: TypeAlias = t.Mapping[str, t.List[t.Mapping[str, t.Any]]]
@@ -142,7 +141,7 @@ def __init__(self, project: Project, path: Path, check_schema: bool) -> None:
         self._meltano_file = self.project.meltanofile.read_text()
         self.path = path
         self.check_schema = check_schema
-        with open(MANIFEST_SCHEMA_PATH) as manifest_schema_file:
+        with MANIFEST_SCHEMA_PATH.open() as manifest_schema_file:
             manifest_schema = json.load(manifest_schema_file)
         self._env_locations = meltano_config_env_locations(manifest_schema)
 

src/meltano/core/plugin/meltano_file.py

@@ -33,7 +33,7 @@ def file_contents(
         Returns:
             A mapping.
         """
-        with (bundle.root / "initialize.yml").open() as f:
+        with bundle.root.joinpath("initialize.yml").open() as f:
             return {
                 Path(relative_path): content
                 for relative_path, content in yaml.safe_load(f).items()

src/meltano/core/tracking/contexts/environment.py

@@ -10,7 +10,6 @@
 from contextlib import suppress
 from datetime import datetime
 from functools import cached_property
-from pathlib import Path
 from warnings import warn
 
 import psutil
@@ -20,11 +19,14 @@
 import meltano
 from meltano.core.tracking.schemas import EnvironmentContextSchema
 from meltano.core.utils import get_boolean_env_var, hash_sha256, safe_hasattr, strtobool
+from meltano.core.utils.compat import importlib_resources
 
 logger = get_logger(__name__)
 
 # This file is only ever created in CI when building a release
-release_marker_path = Path(__file__).parent / ".release_marker"
+release_marker_path = (
+    importlib_resources.files("meltano.core.tracking") / ".release_marker"
+)
 
 
 def _get_parent_context_uuid_str() -> str | None:
@@ -63,14 +65,14 @@ class EnvironmentContext(SelfDescribingJson):
     }
 
     @classmethod
-    def _notable_hashed_env_vars(cls) -> t.Iterable[str]:
+    def _notable_hashed_env_vars(cls) -> t.Iterable[tuple[str, str]]:
         for env_var_name in cls.notable_hashed_env_vars:
             with suppress(KeyError):  # Skip unset env vars
                 env_var_value = os.environ[env_var_name]
                 yield env_var_name, hash_sha256(env_var_value)
 
     @classmethod
-    def _notable_flag_env_vars(cls) -> t.Iterable[str]:
+    def _notable_flag_env_vars(cls) -> t.Iterable[tuple[str, bool | None]]:
         for env_var_name in cls.notable_flag_env_vars:
             with suppress(KeyError):  # Skip unset env vars
                 env_var_value = os.environ[env_var_name]
@@ -87,7 +89,7 @@ def __init__(self):
                 "context_uuid": str(uuid.uuid4()),
                 "parent_context_uuid": _get_parent_context_uuid_str(),
                 "meltano_version": meltano.__version__,
-                "is_dev_build": not release_marker_path.exists(),
+                "is_dev_build": not release_marker_path.is_file(),
                 "is_ci_environment": any(
                     get_boolean_env_var(marker) for marker in self.ci_markers
                 ),

src/meltano/core/upgrade_service.py

@@ -2,14 +2,15 @@
 
 from __future__ import annotations
 
+import json
 import os
 import subprocess
 import sys
 import typing as t
+from importlib.metadata import distribution
 
 import click
 
-import meltano
 from meltano.cli.utils import PluginInstallReason, install_plugins
 from meltano.core.error import MeltanoError
 from meltano.core.project_plugins_service import PluginType
@@ -22,6 +23,50 @@
     from meltano.core.project import Project
 
 
+def _get_pep610_data() -> dict[str, t.Any] | None:
+    dist = distribution("meltano")
+    if contents := dist.read_text("direct_url.json"):
+        return json.loads(contents)
+
+    return None
+
+
+def _check_editable_installation(*, force: bool) -> None:
+    pep610_data = _get_pep610_data()
+
+    if pep610_data:
+        url: str | None = pep610_data.get("url")
+        dir_info: dict[str, t.Any] = pep610_data.get("dir_info", {})
+        if url and dir_info and dir_info.get("editable", False) and not force:
+            meltano_dir = url.removeprefix("file://")
+            raise AutomaticPackageUpgradeError(
+                reason="it is installed from source",
+                instructions=f"navigate to `{meltano_dir}` and run `git pull`",
+            )
+
+
+def _check_docker_installation() -> None:
+    if os.path.exists("/.dockerenv"):
+        raise AutomaticPackageUpgradeError(
+            reason="it is installed inside Docker",
+            instructions=(
+                "pull the latest Docker image using "
+                "`docker pull meltano/meltano` and recreate any containers "
+                "you may have created"
+            ),
+        )
+
+
+def _check_in_nox_session() -> None:
+    if os.getenv("NOX_CURRENT_SESSION") == "tests":
+        raise AutomaticPackageUpgradeError(
+            reason="it is installed inside a Nox test session",
+            instructions=(
+                "run `nox -s tests` to upgrade your project based on the latest version"
+            ),
+        )
+
+
 class UpgradeError(Exception):
     """The Meltano upgrade fails."""
 
@@ -33,7 +78,7 @@ def __init__(self, reason: str, instructions: str):
         """Initialize the `AutomaticPackageUpgradeError`.
 
         Args:
-            reason: The reason the exception occured.
+            reason: The reason the exception occurred.
             instructions: Instructions for how to manually resolve the exception.
         """
         self.reason = reason
@@ -54,33 +99,9 @@ def __init__(self, engine: Engine, project: Project):
         self.engine = engine
 
     def _upgrade_package(self, pip_url: str | None, force: bool) -> bool:
-        fail_reason = None
-        instructions = ""
-
-        meltano_file_path = "/src/meltano/__init__.py"
-        editable = meltano.__file__.endswith(meltano_file_path)
-        if editable and not force:
-            meltano_dir = meltano.__file__[: -len(meltano_file_path)]
-            fail_reason = "it is installed from source"
-            instructions = f"navigate to `{meltano_dir}` and run `git pull`"
-
-        elif os.path.exists("/.dockerenv"):
-            fail_reason = "it is installed inside Docker"
-            instructions = (
-                "pull the latest Docker image using "
-                "`docker pull meltano/meltano` and recreate any containers "
-                "you may have created"
-            )
-
-        elif os.getenv("NOX_CURRENT_SESSION") == "tests":
-            fail_reason = "it is installed inside a Nox test session"
-            instructions = ""
-
-        if fail_reason:
-            raise AutomaticPackageUpgradeError(
-                reason=fail_reason,
-                instructions=instructions,
-            )
+        _check_editable_installation(force=force)
+        _check_docker_installation()
+        _check_in_nox_session()
 
         pip_url = pip_url or "meltano"
         run = subprocess.run(
@@ -168,7 +189,7 @@ def migrate_state(self):
         manager = state_service.state_store_manager
         if isinstance(manager, CloudStateStoreManager):
             click.secho("Applying migrations to project state...", fg="blue")
-            for filepath in state_service.state_store_manager.list_all_files():
+            for filepath in manager.list_all_files():
                 parts = filepath.split(manager.delimiter)
                 if (
                     parts[-1] == "state.json"

src/meltano/core/utils/compat.py

@@ -0,0 +1,12 @@
+"""Compatibility utilities."""
+
+from __future__ import annotations
+
+import sys
+
+if sys.version_info < (3, 9):
+    import importlib_resources
+else:
+    from importlib import resources as importlib_resources
+
+__all__ = ["importlib_resources"]

tests/fixtures/docker/__init__.py

@@ -2,10 +2,10 @@
 
 from __future__ import annotations
 
-from pathlib import Path
-
 import pytest
 
+from meltano.core.utils.compat import importlib_resources
+
 from .snowplow import SnowplowMicro, snowplow, snowplow_optional, snowplow_session
 
 __all__ = [
@@ -24,4 +24,4 @@ def docker_compose_file() -> str:
     Returns:
         The absolute path to the `docker-compose.yml` file used by `pytest-docker`.
     """
-    return str(Path(__file__).parent.resolve() / "docker-compose.yml")
+    return str(importlib_resources.files(__package__) / "docker-compose.yml")

tests/meltano/cli/test_compile.py

@@ -11,7 +11,6 @@
 import mock
 import pytest
 
-from meltano import __file__ as meltano_init_file
 from meltano.cli import cli
 from meltano.core.manifest import manifest
 
@@ -21,7 +20,7 @@
 
     from meltano.core.project import Project
 
-schema_path = Path(meltano_init_file).parent / "schemas" / "meltano.schema.json"
+schema_path = manifest.MANIFEST_SCHEMA_PATH
 
 
 def check_indent(json_path: Path, indent: int):

tests/meltano/cli/test_upgrade.py

@@ -10,7 +10,6 @@
 import pytest
 from moto import mock_s3
 
-import meltano
 from asserts import assert_cli_runner
 from meltano.cli import cli
 
@@ -20,13 +19,19 @@
 
 class TestCliUpgrade:
     @pytest.mark.usefixtures("project")
+    @pytest.mark.xfail(
+        platform.system() == "Windows",
+        reason="Fails on Windows: https://github.com/meltano/meltano/issues/3444",
+    )
     def test_upgrade(self, cli_runner: CliRunner):
-        if platform.system() == "Windows":
-            pytest.xfail(
-                "Fails on Windows: https://github.com/meltano/meltano/issues/3444",
-            )
         # If an editable install was used, test that it cannot be upgraded automatically
-        if meltano.__file__.endswith("/src/meltano/__init__.py"):
+        with mock.patch("importlib.metadata.distribution") as mock_dist:
+            mock_dist.return_value.read_text.return_value = json.dumps(
+                {
+                    "dir_info": {"editable": True},
+                    "url": "file:///Users/user/Code/meltano/meltano",
+                },
+            )
             result = cli_runner.invoke(cli, ["upgrade"])
             assert_cli_runner(result)
 
@@ -56,13 +61,19 @@ def test_upgrade_skip_package(self, cli_runner: CliRunner):
         assert "Your Meltano project has been upgraded!" in result.stdout
 
     @pytest.mark.usefixtures("project")
+    @pytest.mark.xfail(
+        platform.system() == "Windows",
+        reason="Fails on Windows: https://github.com/meltano/meltano/issues/3444",
+    )
     def test_upgrade_package(self, cli_runner: CliRunner):
-        if platform.system() == "Windows":
-            pytest.xfail(
-                "Fails on Windows: https://github.com/meltano/meltano/issues/3444",
-            )
         # If an editable install was used, test that it cannot be upgraded automatically
-        if meltano.__file__.endswith("/src/meltano/__init__.py"):
+        with mock.patch("importlib.metadata.distribution") as mock_dist:
+            mock_dist.return_value.read_text.return_value = json.dumps(
+                {
+                    "dir_info": {"editable": True},
+                    "url": "file:///Users/user/Code/meltano/meltano",
+                },
+            )
             result = cli_runner.invoke(cli, ["upgrade", "package"])
             assert_cli_runner(result)
 
@@ -74,11 +85,11 @@ def test_upgrade_package(self, cli_runner: CliRunner):
 
     @pytest.mark.order(before="test_upgrade_files_glob_path")
     @pytest.mark.usefixtures("session")
+    @pytest.mark.xfail(
+        platform.system() == "Windows",
+        reason="Fails on Windows: https://github.com/meltano/meltano/issues/3444",
+    )
     def test_upgrade_files(self, project, cli_runner: CliRunner):
-        if platform.system() == "Windows":
-            pytest.xfail(
-                "Fails on Windows: https://github.com/meltano/meltano/issues/3444",
-            )
         result = cli_runner.invoke(cli, ["upgrade", "files"])
         output = result.stdout + result.stderr
         assert_cli_runner(result)
@@ -170,12 +181,11 @@ def test_upgrade_files(self, project, cli_runner: CliRunner):
         assert "Updated orchestrate/dags/meltano.py" in output
 
     @pytest.mark.usefixtures("session")
+    @pytest.mark.xfail(
+        platform.system() == "Windows",
+        reason="Fails on Windows: https://github.com/meltano/meltano/issues/3444",
+    )
     def test_upgrade_files_glob_path(self, project, cli_runner: CliRunner):
-        if platform.system() == "Windows":
-            pytest.xfail(
-                "Fails on Windows: https://github.com/meltano/meltano/issues/3444",
-            )
-
         result = cli_runner.invoke(cli, ["add", "files", "airflow"])
         assert_cli_runner(result)