feat: Support uv as an optional virtualenv backend (meltano#8465)

* feat: Support `uv` as an optional `venv` backend

* Use `utf-8` instead of `unicode_replace`

* Add a copyright notice to the `UvVenvService._find_uv` function

* Fix types

* Rename `upgrade_pip` → `upgrade_installer`

* Fail if uv can not be found

docker/meltano/Dockerfile

@@ -22,6 +22,6 @@ ENV PATH="/venv/bin:${PATH}"
 
 # Installing the application the same way our users do when using PyPI
 RUN pip install --upgrade pip wheel && \
-    pip install "meltano[azure,gcs,mssql,postgres,psycopg2,s3]==${MELTANO_VERSION}"
+    pip install "meltano[azure,gcs,mssql,postgres,psycopg2,s3,uv]==${MELTANO_VERSION}"
 
 ENTRYPOINT ["meltano"]

docs/docs/concepts/python_virtual_environments.md

@@ -5,7 +5,7 @@ layout: doc
 sidebar_position: 12
 ---
 
-# What are Python Virtual Environments
+## What are Python Virtual Environments
 
 Python Virtual Environments, also known as a venv, are a way to allow a
 Python application access to specific versions of the libraries it needs to run properly. In the Singer ecosystem
@@ -20,7 +20,7 @@ We're going to focus on the Meltano use case for Virtual Environments which is u
 1. Installing Meltano
 1. Installing Plugins (Taps, Targets, Transformers, etc)
 
-# Why should I care about Virtual Environments?
+## Why should I care about Virtual Environments?
 
 Ideally, you don't have to to worry about Virtual Environments while using Meltano,
 we recommend using pipx to install Meltano which manages the creation of venvs for you.
@@ -30,7 +30,7 @@ However, if you ever need to customize or build your own production pipeline (or
 how to install Meltano in an isolated way so that you don't have a conflict with dependecies within your own Operating System
 or other Python applications running on the same machine.
 
-# How do I use Virtual Environments?
+## How do I use Virtual Environments?
 
 We strongly suggest you create a directory where you want your virtual environments to be saved (e.g. `.venv/meltano/`).
 This can be any directory in your environment, but we recommend saving it in your Meltano project to make it easier to keep
@@ -48,7 +48,7 @@ And that's it! You've created a virtual environment.
 Feel free to explore the directory and compare it to your global python directory!
 You can navigate to the directory by running `cd .venv/meltano/`.
 
-## Activating Your Virtual Environment
+### Activating Your Virtual Environment
 
 Activate the virtual environment, and upgrade pip using:
 
@@ -86,7 +86,7 @@ pip install meltano
 Ensure you have `pipx` installed by reviewing the <a href="/getting-started/installation#install-pipx">Install pipx</a> instructions.</p>
 :::
 
-# How does meltano use Virtual Environments internally?
+## How does meltano use Virtual Environments internally?
 
 Whenever you run `meltano install`, Meltano creates a `.meltano/` directory in your project.
 This directory has a number of sub-directories (subject to change at any point, as this is an

docs/docs/guide/advanced-topics.md

@@ -25,6 +25,18 @@ The following extras add support for other [state backends](/concepts/state_back
 - `gcs` - Google Cloud Storage.
 - `azure` - Azure Blob Storage.
 
+### Virtual Environment backends
+
+The following extras add support for different [virtual environment backends](/concepts/python_virtual_environments#how-does-meltano-use-virtual-environments-internally):
+
+- `uv` - Manage virtual environments with [astral.sh/uv](https://github.com/astral-sh/uv/).
+
+:::info
+  <p>
+  After you switch to a different virtual environment backend, it's recommended to reinstall all plugins with `meltano install --clean`.
+  </p>
+:::
+
 ## Extension Developer Kit (EDK)
 
 Meltano extensions are lightweight executables which allow you to integrate existing data tools with Meltano.

docs/docs/reference/settings.mdx

@@ -876,6 +876,14 @@ export MELTANO_STATE_BACKEND_GCS_APPLICATION_CREDENTIALS="path/to/creds.json"
   </TabItem>
 </Tabs>
 
+## Virtual environments
+
+### `venv.backend`
+
+- [Environment variable](/guide/configuration#configuring-settings): `MELTANO_VENV_BACKEND`
+- Options: `virtualenv`, `uv`
+- Default: `virtualenv`
+
 ## Snowplow Tracking
 
 ### `snowplow.collector_endpoints`

integration/meltano-manifest/expected-manifests/meltano-manifest.cicd.json

@@ -60,7 +60,8 @@
     "MELTANO_SNOWPLOW_COLLECTOR_ENDPOINTS": "[\"https://sp.meltano.com\"]",
     "MELTANO_STATE_BACKEND_LOCK_RETRY_SECONDS": "1",
     "MELTANO_STATE_BACKEND_LOCK_TIMEOUT_SECONDS": "10",
-    "MELTANO_STATE_BACKEND_URI": "systemdb"
+    "MELTANO_STATE_BACKEND_URI": "systemdb",
+    "MELTANO_VENV_BACKEND": "virtualenv"
   },
   "jobs": [
     {

integration/meltano-manifest/expected-manifests/meltano-manifest.jigsaw.json

@@ -46,6 +46,7 @@
     "MELTANO_STATE_BACKEND_LOCK_RETRY_SECONDS": "1",
     "MELTANO_STATE_BACKEND_LOCK_TIMEOUT_SECONDS": "10",
     "MELTANO_STATE_BACKEND_URI": "systemdb",
+    "MELTANO_VENV_BACKEND": "virtualenv",
     "SUPERSET_API_URL": "http://localhost:8088",
     "SUPERSET_PASS": "admin",
     "SUPERSET_USER": "admin",

integration/meltano-manifest/expected-manifests/meltano-manifest.json

@@ -40,7 +40,8 @@
     "MELTANO_SNOWPLOW_COLLECTOR_ENDPOINTS": "[\"https://sp.meltano.com\"]",
     "MELTANO_STATE_BACKEND_LOCK_RETRY_SECONDS": "1",
     "MELTANO_STATE_BACKEND_LOCK_TIMEOUT_SECONDS": "10",
-    "MELTANO_STATE_BACKEND_URI": "systemdb"
+    "MELTANO_STATE_BACKEND_URI": "systemdb",
+    "MELTANO_VENV_BACKEND": "virtualenv"
   },
   "jobs": [
     {

integration/meltano-manifest/expected-manifests/meltano-manifest.prod.json

@@ -42,6 +42,7 @@
     "MELTANO_STATE_BACKEND_LOCK_RETRY_SECONDS": "1",
     "MELTANO_STATE_BACKEND_LOCK_TIMEOUT_SECONDS": "10",
     "MELTANO_STATE_BACKEND_URI": "systemdb",
+    "MELTANO_VENV_BACKEND": "virtualenv",
     "PERMISSION_BOT_DATABASE": "SNOWFLAKE_SAMPLE_DATA",
     "PERMISSION_BOT_ROLE": "SECURITYADMIN",
     "PERMISSION_BOT_USER": "permission_bot",

integration/meltano-manifest/expected-manifests/meltano-manifest.staging.json

@@ -41,7 +41,8 @@
     "MELTANO_SNOWPLOW_COLLECTOR_ENDPOINTS": "[\"https://sp.meltano.com\"]",
     "MELTANO_STATE_BACKEND_LOCK_RETRY_SECONDS": "1",
     "MELTANO_STATE_BACKEND_LOCK_TIMEOUT_SECONDS": "10",
-    "MELTANO_STATE_BACKEND_URI": "systemdb"
+    "MELTANO_STATE_BACKEND_URI": "systemdb",
+    "MELTANO_VENV_BACKEND": "virtualenv"
   },
   "jobs": [
     {

integration/meltano-manifest/expected-manifests/meltano-manifest.userdev.json

@@ -49,6 +49,7 @@
     "MELTANO_STATE_BACKEND_LOCK_RETRY_SECONDS": "1",
     "MELTANO_STATE_BACKEND_LOCK_TIMEOUT_SECONDS": "10",
     "MELTANO_STATE_BACKEND_URI": "systemdb",
+    "MELTANO_VENV_BACKEND": "virtualenv",
     "SUPERSET_API_URL": "http://localhost:8088",
     "SUPERSET_PASS": "admin",
     "SUPERSET_USER": "admin",

noxfile.py

@@ -100,7 +100,7 @@ def pytest_meltano(session: Session) -> None:
         session: Nox session.
     """
     backend_db = os.environ.get("PYTEST_BACKEND", "sqlite")
-    extras = ["azure", "gcs", "s3"]
+    extras = ["azure", "gcs", "s3", "uv"]
 
     if backend_db == "mssql":
         extras.append("mssql")
@@ -160,7 +160,7 @@ def mypy(session: Session) -> None:
         session: Nox session.
     """
     session.install(
-        ".[mssql,azure,gcs,s3]",
+        ".[mssql,azure,gcs,s3,uv]",
         "boto3-stubs",
         "mypy",
         "types-croniter",

pyproject.toml

@@ -86,6 +86,7 @@ typing-extensions = "^4.11.0"
 tzlocal = "^5.2"
 # Compatibility issues with boto: https://github.com/boto/botocore/pull/3034
 urllib3 = "<2"
+uv = { version = ">=0.1.24,<0.2", optional = true }
 virtualenv = "^20.25.1"
 yaspin = "^2.3.0"
 
@@ -96,6 +97,7 @@ mssql = ["pymssql"]
 postgres = ["psycopg"]
 psycopg2 = ["psycopg2-binary"]
 s3 = ["boto3"]
+uv = ["uv"]
 
 [tool.poetry.group.dev.dependencies]
 backoff = "^2.1.2"

src/meltano/core/bundle/settings.yml

@@ -96,6 +96,16 @@ settings:
   kind: array
   value: ["https://sp.meltano.com"]
 
+# venv
+- name: venv.backend
+  value: virtualenv
+  kind: options
+  options:
+  - label: virtualenv
+    value: virtualenv
+  - label: uv
+    value: uv
+
 # Feature Flags
 # Global "experimental" flag.
 - name: experimental

src/meltano/core/plugin_install_service.py

@@ -23,7 +23,7 @@
 from meltano.core.plugin.settings_service import PluginSettingsService
 from meltano.core.settings_service import FeatureFlags
 from meltano.core.utils import EnvVarMissingBehavior, expand_env_vars, noop
-from meltano.core.venv_service import VenvService
+from meltano.core.venv_service import UvVenvService, VenvService
 
 if t.TYPE_CHECKING:
     from meltano.core.plugin.project_plugin import ProjectPlugin
@@ -521,15 +521,33 @@ async def install_pip_plugin(
         force: Whether to ignore the Python version required by plugins.
         env: Environment variables to use when expanding the pip install args.
         kwargs: Unused additional arguments for the installation of the plugin.
+
+    Raises:
+        ValueError: If the venv backend is not supported.
     """
     pip_install_args = get_pip_install_args(project, plugin, env=env)
+    backend = project.settings.get("venv.backend", "virtualenv")
+
+    if backend == "virtualenv":
+        service = VenvService(
+            project=project,
+            python=plugin.python,
+            namespace=plugin.type,
+            name=plugin.venv_name,
+        )
+    elif backend == "uv":  # pragma: no cover
+        logger.warning("The uv backend is experimental")
+        service = UvVenvService(
+            project=project,
+            python=plugin.python,
+            namespace=plugin.type,
+            name=plugin.venv_name,
+        )
+    else:  # pragma: no cover
+        msg = f"Unsupported venv backend: {backend}"
+        raise ValueError(msg)
 
-    await VenvService(
-        project=project,
-        python=plugin.python,
-        namespace=plugin.type,
-        name=plugin.venv_name,
-    ).install(
+    await service.install(
         pip_install_args=("--ignore-requires-python", *pip_install_args)
         if force
         else pip_install_args,