diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 83b6b83bc37ad..993ecfac97f63 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,6 +24,8 @@ env: PACKAGE_NAME: ruff PYTHON_VERSION: "3.14" NEXTEST_PROFILE: ci + # Enable mdtests that require external dependencies + MDTEST_EXTERNAL: "1" jobs: determine_changes: diff --git a/Cargo.lock b/Cargo.lock index 6bc8bf881c33c..6bde255074268 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4557,6 +4557,7 @@ dependencies = [ "anyhow", "camino", "colored 3.0.0", + "dunce", "insta", "memchr", "path-slash", diff --git a/crates/ty_python_semantic/mdtest.py b/crates/ty_python_semantic/mdtest.py index c3260637cca0a..2acc6f452b1ec 100644 --- a/crates/ty_python_semantic/mdtest.py +++ b/crates/ty_python_semantic/mdtest.py @@ -37,14 +37,16 @@ class MDTestRunner: mdtest_executable: Path | None console: Console filters: list[str] + enable_external: bool - def __init__(self, filters: list[str] | None = None) -> None: + def __init__(self, filters: list[str] | None, enable_external: bool) -> None: self.mdtest_executable = None self.console = Console() self.filters = [ f.removesuffix(".md").replace("/", "_").replace("-", "_") for f in (filters or []) ] + self.enable_external = enable_external def _run_cargo_test(self, *, message_format: Literal["human", "json"]) -> str: return subprocess.check_output( @@ -120,6 +122,7 @@ def _run_mdtest( CLICOLOR_FORCE="1", INSTA_FORCE_PASS="1", INSTA_OUTPUT="none", + MDTEST_EXTERNAL="1" if self.enable_external else "0", ), capture_output=capture_output, text=True, @@ -266,11 +269,19 @@ def main() -> None: nargs="*", help="Partial paths or mangled names, e.g., 'loops/for.md' or 'loops_for'", ) + parser.add_argument( + "--enable-external", + "-e", + action="store_true", + help="Enable tests with external dependencies", + ) args = parser.parse_args() try: - runner = MDTestRunner(filters=args.filters) + runner = MDTestRunner( + filters=args.filters, enable_external=args.enable_external + ) runner.watch() except KeyboardInterrupt: print() diff --git a/crates/ty_python_semantic/resources/mdtest/external/README.md b/crates/ty_python_semantic/resources/mdtest/external/README.md new file mode 100644 index 0000000000000..a54c31c862b4e --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/external/README.md @@ -0,0 +1,4 @@ +# mdtests with external dependencies + +This directory contains mdtests that make use of external packages. See the mdtest `README.md` for +more information. diff --git a/crates/ty_python_semantic/resources/mdtest/external/attrs.md b/crates/ty_python_semantic/resources/mdtest/external/attrs.md new file mode 100644 index 0000000000000..3b4bc342a6fbd --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/external/attrs.md @@ -0,0 +1,78 @@ +# attrs + +```toml +[environment] +python-version = "3.13" +python-platform = "linux" + +[project] +dependencies = ["attrs==25.4.0"] +``` + +## Basic class (`attr`) + +```py +import attr + +@attr.s +class User: + id: int = attr.ib() + name: str = attr.ib() + +user = User(id=1, name="John Doe") + +reveal_type(user.id) # revealed: int +reveal_type(user.name) # revealed: str +``` + +## Basic class (`define`) + +```py +from attrs import define, field + +@define +class User: + id: int = field() + internal_name: str = field(alias="name") + +user = User(id=1, name="John Doe") +reveal_type(user.id) # revealed: int +reveal_type(user.internal_name) # revealed: str +``` + +## Usage of `field` parameters + +```py +from attrs import define, field + +@define +class Product: + id: int = field(init=False) + name: str = field() + price_cent: int = field(kw_only=True) + +reveal_type(Product.__init__) # revealed: (self: Product, name: str, *, price_cent: int) -> None +``` + +## Dedicated support for the `default` decorator? + +We currently do not support this: + +```py +from attrs import define, field + +@define +class Person: + id: int = field() + name: str = field() + + # error: [call-non-callable] "Object of type `_MISSING_TYPE` is not callable" + @id.default + def _default_id(self) -> int: + raise NotImplementedError + +# error: [missing-argument] "No argument provided for required parameter `id`" +person = Person(name="Alice") +reveal_type(person.id) # revealed: int +reveal_type(person.name) # revealed: str +``` diff --git a/crates/ty_python_semantic/resources/mdtest/external/numpy.md b/crates/ty_python_semantic/resources/mdtest/external/numpy.md new file mode 100644 index 0000000000000..39bfa6d1106d8 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/external/numpy.md @@ -0,0 +1,23 @@ +# numpy + +```toml +[environment] +python-version = "3.13" +python-platform = "linux" + +[project] +dependencies = ["numpy==2.3.0"] +``` + +## Basic usage + +```py +import numpy as np + +xs = np.array([1, 2, 3]) +reveal_type(xs) # revealed: ndarray[tuple[Any, ...], dtype[Any]] + +xs = np.array([1.0, 2.0, 3.0], dtype=np.float64) +# TODO: should be `ndarray[tuple[Any, ...], dtype[float64]]` +reveal_type(xs) # revealed: ndarray[tuple[Any, ...], dtype[Unknown]] +``` diff --git a/crates/ty_python_semantic/resources/mdtest/external/pydantic.md b/crates/ty_python_semantic/resources/mdtest/external/pydantic.md new file mode 100644 index 0000000000000..6fb82840f5788 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/external/pydantic.md @@ -0,0 +1,48 @@ +# Pydantic + +```toml +[environment] +python-version = "3.12" +python-platform = "linux" + +[project] +dependencies = ["pydantic==2.12.2"] +``` + +## Basic model + +```py +from pydantic import BaseModel + +class User(BaseModel): + id: int + name: str + +reveal_type(User.__init__) # revealed: (self: User, *, id: int, name: str) -> None + +user = User(id=1, name="John Doe") +reveal_type(user.id) # revealed: int +reveal_type(user.name) # revealed: str + +# error: [missing-argument] "No argument provided for required parameter `name`" +invalid_user = User(id=2) +``` + +## Usage of `Field` + +```py +from pydantic import BaseModel, Field + +class Product(BaseModel): + id: int = Field(init=False) + name: str = Field(..., kw_only=False, min_length=1) + internal_price_cent: int = Field(..., gt=0, alias="price_cent") + +reveal_type(Product.__init__) # revealed: (self: Product, name: str = Any, *, price_cent: int = Any) -> None + +product = Product("Laptop", price_cent=999_00) + +reveal_type(product.id) # revealed: int +reveal_type(product.name) # revealed: str +reveal_type(product.internal_price_cent) # revealed: int +``` diff --git a/crates/ty_python_semantic/resources/mdtest/external/pytest.md b/crates/ty_python_semantic/resources/mdtest/external/pytest.md new file mode 100644 index 0000000000000..823ef4d162d99 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/external/pytest.md @@ -0,0 +1,27 @@ +# pytest + +```toml +[environment] +python-version = "3.13" +python-platform = "linux" + +[project] +dependencies = ["pytest==9.0.1"] +``` + +## `pytest.fail` + +Make sure that we recognize `pytest.fail` calls as terminal: + +```py +import pytest + +def some_runtime_condition() -> bool: + return True + +def test_something(): + if not some_runtime_condition(): + pytest.fail("Runtime condition failed") + + no_error_here_this_is_unreachable +``` diff --git a/crates/ty_python_semantic/resources/mdtest/external/sqlalchemy.md b/crates/ty_python_semantic/resources/mdtest/external/sqlalchemy.md new file mode 100644 index 0000000000000..0ac9c4c219827 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/external/sqlalchemy.md @@ -0,0 +1,124 @@ +# SQLAlchemy + +```toml +[environment] +python-version = "3.13" +python-platform = "linux" + +[project] +dependencies = ["SQLAlchemy==2.0.44"] +``` + +## Basic model + +Here, we mostly make sure that ty understands SQLAlchemy's dataclass-transformer setup: + +```py +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + +class Base(DeclarativeBase): + pass + +class User(Base): + __tablename__ = "user" + + id: Mapped[int] = mapped_column(primary_key=True, init=False) + internal_name: Mapped[str] = mapped_column(alias="name") + +user = User(name="John Doe") +reveal_type(user.id) # revealed: int +reveal_type(user.internal_name) # revealed: str +``` + +Unfortunately, SQLAlchemy overrides `__init__` and explicitly accepts all combinations of keyword +arguments. This is why we currently cannot flag invalid constructor calls: + +```py +reveal_type(User.__init__) # revealed: def __init__(self, **kw: Any) -> Unknown + +# TODO: this should ideally be an error +invalid_user = User(invalid_arg=42) +``` + +## Queries + +First, the basic setup: + +```py +from datetime import datetime + +from sqlalchemy import select, Integer, Text, Boolean, DateTime +from sqlalchemy.orm import Session +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy import create_engine + +engine = create_engine("sqlite://example.db") +session = Session(engine) +``` + +Now we can declare a simple model: + +```py +class Base(DeclarativeBase): + pass + +class User(Base): + __tablename__ = "users" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + name: Mapped[str] = mapped_column(Text) + is_admin: Mapped[bool] = mapped_column(Boolean, default=False) +``` + +And perform simple queries: + +```py +stmt = select(User) +reveal_type(stmt) # revealed: Select[tuple[User]] + +users = session.scalars(stmt).all() +reveal_type(users) # revealed: Sequence[User] + +for row in session.execute(stmt): + reveal_type(row) # revealed: Row[tuple[User]] + +stmt = select(User).where(User.name == "Alice") +alice = session.scalars(stmt).first() +reveal_type(alice) # revealed: User | None + +stmt = select(User).where(User.is_admin == True).order_by(User.name).limit(10) +admin_users = session.scalars(stmt).all() +reveal_type(admin_users) # revealed: Sequence[User] +``` + +This also works with the legacy `query` API: + +```py +users_legacy = session.query(User).all() +reveal_type(users_legacy) # revealed: list[User] +``` + +We can also specify particular columns to select: + +```py +stmt = select(User.id, User.name) +# TODO: should be `Select[tuple[int, str]]` +reveal_type(stmt) # revealed: Select[tuple[Unknown, Unknown]] + +for row in session.execute(stmt): + # TODO: should be `Row[Tuple[int, str]]` + reveal_type(row) # revealed: Row[tuple[Unknown, Unknown]] +``` + +And similarly with the legacy `query` API: + +```py +query = session.query(User.id, User.name) +# TODO: should be `RowReturningQuery[tuple[int, str]]` +reveal_type(query) # revealed: RowReturningQuery[tuple[Unknown, Unknown]] + +for row in query.all(): + # TODO: should be `Row[Tuple[int, str]]` + reveal_type(row) # revealed: Row[tuple[Unknown, Unknown]] +``` diff --git a/crates/ty_python_semantic/resources/mdtest/external/sqlmodel.md b/crates/ty_python_semantic/resources/mdtest/external/sqlmodel.md new file mode 100644 index 0000000000000..7dafa336db2fe --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/external/sqlmodel.md @@ -0,0 +1,30 @@ +# SQLModel + +```toml +[environment] +python-version = "3.13" +python-platform = "linux" + +[project] +dependencies = ["sqlmodel==0.0.27"] +``` + +## Basic model + +```py +from sqlmodel import SQLModel + +class User(SQLModel): + id: int + name: str + +user = User(id=1, name="John Doe") +reveal_type(user.id) # revealed: int +reveal_type(user.name) # revealed: str + +# TODO: this should not mention `__pydantic_self__`, and have proper parameters defined by the fields +reveal_type(User.__init__) # revealed: def __init__(__pydantic_self__, **data: Any) -> None + +# TODO: this should be an error +User() +``` diff --git a/crates/ty_python_semantic/resources/mdtest/external/strawberry.md b/crates/ty_python_semantic/resources/mdtest/external/strawberry.md new file mode 100644 index 0000000000000..849b50aa74d29 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/external/strawberry.md @@ -0,0 +1,27 @@ +# Strawberry GraphQL + +```toml +[environment] +python-version = "3.13" +python-platform = "linux" + +[project] +dependencies = ["strawberry-graphql==0.283.3"] +``` + +## Basic model + +```py +import strawberry + +@strawberry.type +class User: + id: int + role: str = strawberry.field(default="user") + +reveal_type(User.__init__) # revealed: (self: User, *, id: int, role: str = Any) -> None + +user = User(id=1) +reveal_type(user.id) # revealed: int +reveal_type(user.role) # revealed: str +``` diff --git a/crates/ty_test/Cargo.toml b/crates/ty_test/Cargo.toml index f300b614a0d6d..a7c18b9d002d7 100644 --- a/crates/ty_test/Cargo.toml +++ b/crates/ty_test/Cargo.toml @@ -25,6 +25,7 @@ ty_vendored = { workspace = true } anyhow = { workspace = true } camino = { workspace = true } +dunce = { workspace = true } colored = { workspace = true } insta = { workspace = true, features = ["filters"] } memchr = { workspace = true } diff --git a/crates/ty_test/README.md b/crates/ty_test/README.md index ecf4614d94d14..b31a45e87774f 100644 --- a/crates/ty_test/README.md +++ b/crates/ty_test/README.md @@ -316,6 +316,44 @@ To enable logging in an mdtest, set `log = true` at the top level of the TOML bl See [`MarkdownTestConfig`](https://github.com/astral-sh/ruff/blob/main/crates/ty_test/src/config.rs) for the full list of supported configuration options. +### Testing with external dependencies + +Tests can specify external Python dependencies using a `[project]` section in the TOML configuration. +This allows testing code that uses third-party libraries like `pydantic`, `numpy`, etc. + +It is recommended to specify exact versions of packages to ensure reproducibility. The specified +Python version and platform are required for tests with external dependencies, as they are used +during package resolution. + +````markdown +```toml +[environment] +python-version = "3.13" +python-platform = "linux" + +[project] +dependencies = ["pydantic==2.12.2"] +``` + +```py +import pydantic + +# use pydantic in the test +``` +```` + +When a test has dependencies: + +1. The test framework creates a `pyproject.toml` in a temporary directory. +1. Runs `uv sync` to install the dependencies. +1. Copies the installed packages from the virtual environment's `site-packages` directory into the test's + in-memory filesystem. +1. Configures the type checker to use these packages. + +**Note**: This feature requires `uv` to be installed and available in your `PATH`. The dependencies +are installed fresh for each test that specifies them, so tests with many dependencies may be slower +to run. + ### Specifying a custom typeshed Some tests will need to override the default typeshed with custom files. The `[environment]` diff --git a/crates/ty_test/src/config.rs b/crates/ty_test/src/config.rs index 6fe3a17fd0537..6154208d69f26 100644 --- a/crates/ty_test/src/config.rs +++ b/crates/ty_test/src/config.rs @@ -4,8 +4,12 @@ //! //! ```toml //! log = true # or log = "ty=WARN" +//! //! [environment] //! python-version = "3.10" +//! +//! [project] +//! dependencies = ["pydantic==2.12.2"] //! ``` use anyhow::Context; @@ -25,6 +29,9 @@ pub(crate) struct MarkdownTestConfig { /// /// Defaults to the case-sensitive [`ruff_db::system::InMemorySystem`]. pub(crate) system: Option, + + /// Project configuration for installing external dependencies. + pub(crate) project: Option, } impl MarkdownTestConfig { @@ -51,6 +58,10 @@ impl MarkdownTestConfig { pub(crate) fn python(&self) -> Option<&SystemPath> { self.environment.as_ref()?.python.as_deref() } + + pub(crate) fn dependencies(&self) -> Option<&[String]> { + self.project.as_ref()?.dependencies.as_deref() + } } #[derive(Deserialize, Debug, Default, Clone)] @@ -116,3 +127,16 @@ pub(crate) enum SystemKind { /// This system should only be used when testing system or OS specific behavior. Os, } + +/// Project configuration for tests that need external dependencies. +#[derive(Deserialize, Debug, Default, Clone)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub(crate) struct Project { + /// List of Python package dependencies in `pyproject.toml` format. + /// + /// These will be installed using `uv sync` into a temporary virtual environment. + /// The site-packages directory will then be copied into the test's filesystem. + /// + /// Example: `dependencies = ["pydantic==2.12.2"]` + pub(crate) dependencies: Option>, +} diff --git a/crates/ty_test/src/external_dependencies.rs b/crates/ty_test/src/external_dependencies.rs new file mode 100644 index 0000000000000..38a14d78c9782 --- /dev/null +++ b/crates/ty_test/src/external_dependencies.rs @@ -0,0 +1,186 @@ +use crate::db::Db; + +use anyhow::{Context, Result, anyhow, bail}; +use ruff_db::system::{DbWithWritableSystem as _, OsSystem, SystemPath}; +use ruff_python_ast::PythonVersion; +use ty_python_semantic::{PythonEnvironment, PythonPlatform, SysPrefixPathOrigin}; + +/// Setup a virtual environment in the in-memory filesystem of `db` with +/// the specified dependencies installed. +pub(crate) fn setup_venv( + db: &mut Db, + dependencies: &[String], + python_version: PythonVersion, + python_platform: &PythonPlatform, + dest_venv_path: &SystemPath, +) -> Result<()> { + // Create a temporary directory for the project + let temp_dir = tempfile::Builder::new() + .prefix("mdtest-venv-") + .tempdir() + .context("Failed to create temporary directory for mdtest virtual environment")?; + + // Canonicalize here to fix problems with `.strip_prefix()` later on Windows + let temp_dir_path = dunce::canonicalize(temp_dir.path()) + .context("Failed to canonicalize temporary directory path")?; + + let temp_path = SystemPath::from_std_path(&temp_dir_path) + .ok_or_else(|| { + anyhow!( + "Temporary directory path is not valid UTF-8: {}", + temp_dir_path.display() + ) + })? + .to_path_buf(); + + // Generate a minimal pyproject.toml + let pyproject_toml = format!( + r#"[project] +name = "mdtest-deps" +version = "0.1.0" +requires-python = "~={python_version}.0" +dependencies = [ +{deps} +] +"#, + python_version = python_version, + deps = dependencies + .iter() + .map(|dep| format!(" \"{dep}\",")) + .collect::>() + .join("\n") + ); + + std::fs::write( + temp_path.join("pyproject.toml").as_std_path(), + pyproject_toml, + ) + .context("Failed to write pyproject.toml")?; + + // Convert PythonPlatform to uv's platform format + let uv_platform = match python_platform { + PythonPlatform::Identifier(id) => match id.as_str() { + "win32" => "windows", + "darwin" => "macos", + "linux" => "linux", + other => other, + }, + PythonPlatform::All => { + bail!("For an mdtest with external dependencies, a Python platform must be specified"); + } + }; + + // Run `uv sync` to install dependencies + let uv_sync_output = std::process::Command::new("uv") + .args(["sync", "--python-platform", uv_platform]) + .current_dir(temp_path.as_std_path()) + .output() + .context("Failed to run `uv sync`. Is `uv` installed?")?; + + if !uv_sync_output.status.success() { + let stderr = String::from_utf8_lossy(&uv_sync_output.stderr); + bail!( + "`uv sync` failed with exit code {:?}:\n{}", + uv_sync_output.status.code(), + stderr + ); + } + + let venv_path = temp_path.join(".venv"); + + copy_site_packages_to_db(db, &venv_path, dest_venv_path, python_version) +} + +/// Copy the site-packages directory from a real virtual environment to the in-memory filesystem of `db`. +/// +/// This recursively copies all files from the venv's site-packages directory into the +/// in-memory filesystem at the specified destination path. +fn copy_site_packages_to_db( + db: &mut Db, + venv_path: &SystemPath, + dest_venv_path: &SystemPath, + _python_version: PythonVersion, +) -> Result<()> { + // Discover the site-packages directory in the virtual environment + let system = OsSystem::new(venv_path); + let env = PythonEnvironment::new(venv_path, SysPrefixPathOrigin::LocalVenv, &system) + .context("Failed to create Python environment for temporary virtual environment")?; + + let site_packages_paths = env + .site_packages_paths(&system) + .context(format!("Failed to discover site-packages in '{venv_path}'"))?; + + let site_packages_path = site_packages_paths + .into_iter() + .next() + .ok_or_else(|| anyhow!("No site-packages directory found in '{venv_path}'"))?; + + // Create the destination directory structure + let relative_site_packages = site_packages_path.strip_prefix(venv_path).map_err(|_| { + anyhow!("site-packages path '{site_packages_path}' is not under venv path '{venv_path}'") + })?; + let dest_site_packages = dest_venv_path.join(relative_site_packages); + db.create_directory_all(&dest_site_packages) + .context("Failed to create site-packages directory in database")?; + + // Recursively copy all files from site-packages + copy_directory_recursive(db, &site_packages_path, &dest_site_packages)?; + + Ok(()) +} + +fn copy_directory_recursive(db: &mut Db, src: &SystemPath, dest: &SystemPath) -> Result<()> { + use std::fs; + + for entry in fs::read_dir(src.as_std_path()) + .with_context(|| format!("Failed to read directory {src}"))? + { + let entry = entry.with_context(|| format!("Failed to read directory entry in {src}"))?; + let entry_path = entry.path(); + let file_type = entry + .file_type() + .with_context(|| format!("Failed to get file type for {}", entry_path.display()))?; + + let src_path = SystemPath::from_std_path(&entry_path) + .ok_or_else(|| anyhow!("Path {} is not valid UTF-8", entry_path.display()))?; + + let file_name = entry.file_name(); + let file_name_str = file_name.to_str().ok_or_else(|| { + anyhow!( + "File name {} is not valid UTF-8", + file_name.to_string_lossy() + ) + })?; + + let dest_path = dest.join(file_name_str); + + if file_type.is_dir() { + // Skip __pycache__ directories and other unnecessary directories + if file_name_str == "__pycache__" || file_name_str.ends_with(".dist-info") { + continue; + } + + db.create_directory_all(&dest_path) + .with_context(|| format!("Failed to create directory {dest_path}"))?; + + copy_directory_recursive(db, src_path, &dest_path)?; + } else if file_type.is_file() { + let is_python_source = entry_path.extension().is_some_and(|ext| { + ext.eq_ignore_ascii_case("py") || ext.eq_ignore_ascii_case("pyi") + }); + + if !is_python_source { + // Skip all non-Python files (binaries, data files, etc.) + continue; + } + + let contents = fs::read_to_string(src_path.as_std_path()) + .with_context(|| format!("Failed to read file {src_path}"))?; + + db.write_file(&dest_path, contents) + .with_context(|| format!("Failed to write file {dest_path}"))?; + } + } + + Ok(()) +} diff --git a/crates/ty_test/src/lib.rs b/crates/ty_test/src/lib.rs index feb38bdf663b1..ad492617392d3 100644 --- a/crates/ty_test/src/lib.rs +++ b/crates/ty_test/src/lib.rs @@ -28,6 +28,7 @@ mod assertion; mod config; mod db; mod diagnostic; +mod external_dependencies; mod matcher; mod parser; @@ -70,16 +71,21 @@ pub fn run( Log::Filter(filter) => setup_logging_with_filter(filter), }); - let failures = run_test(&mut db, relative_fixture_path, snapshot_path, &test); - let inconsistencies = run_module_resolution_consistency_test(&db); - let this_test_failed = failures.is_err() || inconsistencies.is_err(); + let result = run_test(&mut db, relative_fixture_path, snapshot_path, &test); + let inconsistencies = if result.as_ref().is_ok_and(|t| t.has_been_skipped()) { + Ok(()) + } else { + run_module_resolution_consistency_test(&db) + }; + + let this_test_failed = result.is_err() || inconsistencies.is_err(); any_failures = any_failures || this_test_failed; if this_test_failed && output_format.is_cli() { println!("\n{}\n", test.name().bold().underline()); } - if let Err(failures) = failures { + if let Err(failures) = result { let md_index = LineIndex::from_source_text(&source); for test_failures in failures { @@ -212,12 +218,24 @@ impl OutputFormat { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TestOutcome { + Success, + Skipped, +} + +impl TestOutcome { + const fn has_been_skipped(self) -> bool { + matches!(self, TestOutcome::Skipped) + } +} + fn run_test( db: &mut db::Db, relative_fixture_path: &Utf8Path, snapshot_path: &Utf8Path, test: &parser::MarkdownTest, -) -> Result<(), Failures> { +) -> Result { // Initialize the system and remove all files and directories to reset the system to a clean state. match test.configuration().system.unwrap_or_default() { SystemKind::InMemory => { @@ -248,6 +266,27 @@ fn run_test( let custom_typeshed_path = test.configuration().typeshed(); let python_version = test.configuration().python_version().unwrap_or_default(); + // Setup virtual environment with dependencies if specified + let venv_for_external_dependencies = SystemPathBuf::from("/.venv"); + if let Some(dependencies) = test.configuration().dependencies() { + if !std::env::var("MDTEST_EXTERNAL").is_ok_and(|v| v == "1") { + return Ok(TestOutcome::Skipped); + } + + let python_platform = test.configuration().python_platform().expect( + "Tests with external dependencies must specify `python-platform` in the configuration", + ); + + external_dependencies::setup_venv( + db, + dependencies, + python_version, + &python_platform, + &venv_for_external_dependencies, + ) + .expect("Failed to setup in-memory virtual environment with dependencies"); + } + let mut typeshed_files = vec![]; let mut has_custom_versions_file = false; @@ -350,7 +389,19 @@ fn run_test( let configuration = test.configuration(); - let site_packages_paths = if let Some(python) = configuration.python() { + let site_packages_paths = if configuration.dependencies().is_some() { + // If dependencies were specified, use the venv we just set up + let environment = PythonEnvironment::new( + &venv_for_external_dependencies, + SysPrefixPathOrigin::PythonCliFlag, + db.system(), + ) + .expect("Python environment to point to a valid path"); + environment + .site_packages_paths(db.system()) + .expect("Python environment to be valid") + .into_vec() + } else if let Some(python) = configuration.python() { let environment = PythonEnvironment::new(python, SysPrefixPathOrigin::PythonCliFlag, db.system()) .expect("Python environment to point to a valid path"); @@ -551,7 +602,7 @@ fn run_test( } if failures.is_empty() { - Ok(()) + Ok(TestOutcome::Success) } else { Err(failures) }