diff --git a/crates/karva/tests/it/main.rs b/crates/karva/tests/it/main.rs index 8f31a426..3115737e 100644 --- a/crates/karva/tests/it/main.rs +++ b/crates/karva/tests/it/main.rs @@ -9,5 +9,6 @@ mod durations; mod extensions; mod filterset; mod last_failed; +mod run_ignored; mod version; mod watch; diff --git a/crates/karva/tests/it/run_ignored.rs b/crates/karva/tests/it/run_ignored.rs new file mode 100644 index 00000000..7b1a9009 --- /dev/null +++ b/crates/karva/tests/it/run_ignored.rs @@ -0,0 +1,193 @@ +use insta_cmd::assert_cmd_snapshot; + +use crate::common::TestContext; + +const MIXED_TESTS: &str = r" +import karva + +@karva.tags.skip +def test_skipped(): + assert False + +@karva.tags.skip('reason here') +def test_skipped_with_reason(): + assert False + +def test_normal(): + assert True +"; + +#[test] +fn runignored_runs_only_skipped_tests() { + let context = TestContext::with_file("test.py", MIXED_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("--run-ignored").arg("only"), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 3 tests across 1 worker + FAIL [TIME] test::test_skipped + FAIL [TIME] test::test_skipped_with_reason + SKIP [TIME] test::test_normal + + diagnostics: + + error[test-failure]: Test `test_skipped` failed + --> test.py:5:5 + | + 4 | @karva.tags.skip + 5 | def test_skipped(): + | ^^^^^^^^^^^^ + 6 | assert False + | + info: Test failed here + --> test.py:6:5 + | + 4 | @karva.tags.skip + 5 | def test_skipped(): + 6 | assert False + | ^^^^^^^^^^^^ + 7 | + 8 | @karva.tags.skip('reason here') + | + + error[test-failure]: Test `test_skipped_with_reason` failed + --> test.py:9:5 + | + 8 | @karva.tags.skip('reason here') + 9 | def test_skipped_with_reason(): + | ^^^^^^^^^^^^^^^^^^^^^^^^ + 10 | assert False + | + info: Test failed here + --> test.py:10:5 + | + 8 | @karva.tags.skip('reason here') + 9 | def test_skipped_with_reason(): + 10 | assert False + | ^^^^^^^^^^^^ + 11 | + 12 | def test_normal(): + | + + ──────────── + Summary [TIME] 3 tests run: 0 passed, 2 failed, 1 skipped + + ----- stderr ----- + "); +} + +#[test] +fn runignored_all_runs_skipped_alongside_normal() { + let context = TestContext::with_file("test.py", MIXED_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("--run-ignored").arg("all"), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 3 tests across 1 worker + FAIL [TIME] test::test_skipped + FAIL [TIME] test::test_skipped_with_reason + PASS [TIME] test::test_normal + + diagnostics: + + error[test-failure]: Test `test_skipped` failed + --> test.py:5:5 + | + 4 | @karva.tags.skip + 5 | def test_skipped(): + | ^^^^^^^^^^^^ + 6 | assert False + | + info: Test failed here + --> test.py:6:5 + | + 4 | @karva.tags.skip + 5 | def test_skipped(): + 6 | assert False + | ^^^^^^^^^^^^ + 7 | + 8 | @karva.tags.skip('reason here') + | + + error[test-failure]: Test `test_skipped_with_reason` failed + --> test.py:9:5 + | + 8 | @karva.tags.skip('reason here') + 9 | def test_skipped_with_reason(): + | ^^^^^^^^^^^^^^^^^^^^^^^^ + 10 | assert False + | + info: Test failed here + --> test.py:10:5 + | + 8 | @karva.tags.skip('reason here') + 9 | def test_skipped_with_reason(): + 10 | assert False + | ^^^^^^^^^^^^ + 11 | + 12 | def test_normal(): + | + + ──────────── + Summary [TIME] 3 tests run: 1 passed, 2 failed, 0 skipped + + ----- stderr ----- + "); +} + +#[test] +fn runignored_with_no_skipped_tests_skips_all() { + let context = TestContext::with_file( + "test.py", + r" +def test_alpha(): + assert True + +def test_beta(): + assert True +", + ); + assert_cmd_snapshot!(context.command_no_parallel().arg("--run-ignored").arg("only"), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 2 tests across 1 worker + SKIP [TIME] test::test_alpha + SKIP [TIME] test::test_beta + + ──────────── + Summary [TIME] 2 tests run: 0 passed, 2 skipped + + ----- stderr ----- + "); +} + +#[test] +fn runignored_skipif_false_not_matched() { + let context = TestContext::with_file( + "test.py", + r" +import karva + +@karva.tags.skip(False, reason='Condition is false') +def test_conditional(): + assert True + +def test_normal(): + assert True +", + ); + assert_cmd_snapshot!(context.command_no_parallel().arg("--run-ignored").arg("only"), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 2 tests across 1 worker + SKIP [TIME] test::test_conditional + SKIP [TIME] test::test_normal + + ──────────── + Summary [TIME] 2 tests run: 0 passed, 2 skipped + + ----- stderr ----- + "); +} diff --git a/crates/karva_cli/src/lib.rs b/crates/karva_cli/src/lib.rs index 79194bfa..12e34713 100644 --- a/crates/karva_cli/src/lib.rs +++ b/crates/karva_cli/src/lib.rs @@ -5,7 +5,7 @@ use clap::Parser; use clap::builder::Styles; use clap::builder::styling::{AnsiColor, Effects}; use karva_logging::{TerminalColor, VerbosityLevel}; -use karva_metadata::{MaxFail, Options, SrcOptions, TerminalOptions, TestOptions}; +use karva_metadata::{MaxFail, Options, RunIgnoredMode, SrcOptions, TerminalOptions, TestOptions}; use ruff_db::diagnostic::DiagnosticFormat; const STYLES: Styles = Styles::styled() @@ -244,6 +244,10 @@ pub struct SubTestCommand { #[clap(short = 'E', long = "filter")] pub filter_expressions: Vec, + /// Run ignored tests. + #[arg(long)] + pub run_ignored: Option, + /// Update snapshots directly instead of creating pending `.snap.new` files. /// /// When set, `karva.assert_snapshot()` will write directly to `.snap` files, @@ -375,3 +379,31 @@ impl TestCommand { self.sub_command.into_options() } } + +/// Whether to run ignored/skipped tests. +#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum RunIgnored { + /// Run only ignored tests. + Only, + + /// Run both ignored and non-ignored tests. + All, +} + +impl RunIgnored { + pub fn as_str(self) -> &'static str { + match self { + Self::Only => "only", + Self::All => "all", + } + } +} + +impl From for RunIgnoredMode { + fn from(value: RunIgnored) -> Self { + match value { + RunIgnored::Only => Self::Only, + RunIgnored::All => Self::All, + } + } +} diff --git a/crates/karva_metadata/src/lib.rs b/crates/karva_metadata/src/lib.rs index b2963fd6..16b34f5a 100644 --- a/crates/karva_metadata/src/lib.rs +++ b/crates/karva_metadata/src/lib.rs @@ -14,7 +14,7 @@ pub use options::{ Options, OutputFormat, ProjectOptionsOverrides, SrcOptions, TerminalOptions, TestOptions, }; pub use pyproject::{PyProject, PyProjectError}; -pub use settings::ProjectSettings; +pub use settings::{ProjectSettings, RunIgnoredMode}; use crate::options::KarvaTomlError; diff --git a/crates/karva_metadata/src/options.rs b/crates/karva_metadata/src/options.rs index 7efd2a00..82701e24 100644 --- a/crates/karva_metadata/src/options.rs +++ b/crates/karva_metadata/src/options.rs @@ -7,7 +7,9 @@ use thiserror::Error; use crate::filter::FiltersetSet; use crate::max_fail::MaxFail; -use crate::settings::{ProjectSettings, SrcSettings, TerminalSettings, TestSettings}; +use crate::settings::{ + ProjectSettings, RunIgnoredMode, SrcSettings, TerminalSettings, TestSettings, +}; #[derive( Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, OptionsMetadata, Combine, @@ -231,6 +233,7 @@ impl TestOptions { try_import_fixtures: self.try_import_fixtures.unwrap_or_default(), retry: self.retry.unwrap_or_default(), filter: FiltersetSet::default(), + run_ignored: RunIgnoredMode::default(), } } } diff --git a/crates/karva_metadata/src/settings.rs b/crates/karva_metadata/src/settings.rs index c118327a..d6c726d8 100644 --- a/crates/karva_metadata/src/settings.rs +++ b/crates/karva_metadata/src/settings.rs @@ -2,6 +2,14 @@ use crate::filter::FiltersetSet; use crate::max_fail::MaxFail; use crate::options::OutputFormat; +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunIgnoredMode { + #[default] + Default, + Only, + All, +} + #[derive(Default, Debug, Clone)] pub struct ProjectSettings { pub(crate) terminal: TerminalSettings, @@ -29,6 +37,10 @@ impl ProjectSettings { pub fn set_filter(&mut self, filter: FiltersetSet) { self.test.filter = filter; } + + pub fn set_run_ignored(&mut self, mode: RunIgnoredMode) { + self.test.run_ignored = mode; + } } #[derive(Default, Debug, Clone)] @@ -50,4 +62,5 @@ pub struct TestSettings { pub try_import_fixtures: bool, pub retry: u32, pub filter: FiltersetSet, + pub run_ignored: RunIgnoredMode, } diff --git a/crates/karva_runner/src/orchestration.rs b/crates/karva_runner/src/orchestration.rs index 41109f41..5032e5e5 100644 --- a/crates/karva_runner/src/orchestration.rs +++ b/crates/karva_runner/src/orchestration.rs @@ -432,6 +432,11 @@ fn inner_cli_args(settings: &ProjectSettings, args: &SubTestCommand) -> Vec>; use karva_diagnostic::IndividualTestResultKind; +use karva_metadata::RunIgnoredMode; use karva_metadata::filter::EvalContext; use karva_python_semantic::{FunctionKind, QualifiedFunctionName, QualifiedTestName}; use pyo3::prelude::*; @@ -212,6 +213,8 @@ impl<'ctx, 'a> PackageRunner<'ctx, 'a> { tags: &crate::extensions::tags::Tags, ) -> Option { let filter = &self.context.settings().test().filter; + let run_ignored = self.context.settings().test().run_ignored; + if !filter.is_empty() { let qualified = QualifiedTestName::new(name.clone(), None); let display_name = qualified.to_string(); @@ -229,12 +232,30 @@ impl<'ctx, 'a> PackageRunner<'ctx, 'a> { } } - if let (true, reason) = tags.should_skip() { - return Some(self.context.register_test_case_result( - &QualifiedTestName::new(name.clone(), None), - IndividualTestResultKind::Skipped { reason }, - std::time::Duration::ZERO, - )); + match run_ignored { + RunIgnoredMode::Default => { + if let (true, reason) = tags.should_skip() { + return Some(self.context.register_test_case_result( + &QualifiedTestName::new(name.clone(), None), + IndividualTestResultKind::Skipped { reason }, + std::time::Duration::ZERO, + )); + } + } + RunIgnoredMode::Only => { + // Skip tests whose skip condition is not active; only tests + // that would actually be skipped in a normal run are included. + if let (false, _) = tags.should_skip() { + return Some(self.context.register_test_case_result( + &QualifiedTestName::new(name.clone(), None), + IndividualTestResultKind::Skipped { reason: None }, + std::time::Duration::ZERO, + )); + } + } + RunIgnoredMode::All => { + // run everything regardless of skip tags + } } None diff --git a/crates/karva_worker/src/cli.rs b/crates/karva_worker/src/cli.rs index acaf4815..eeb02e69 100644 --- a/crates/karva_worker/src/cli.rs +++ b/crates/karva_worker/src/cli.rs @@ -10,6 +10,7 @@ use karva_cache::{Cache, RunHash}; use karva_cli::{SubTestCommand, Verbosity}; use karva_diagnostic::{DummyReporter, Reporter, TestCaseReporter}; use karva_logging::{Printer, set_colored_override, setup_tracing}; +use karva_metadata::RunIgnoredMode; use karva_metadata::filter::FiltersetSet; use karva_project::path::{TestPath, TestPathError, absolute}; use karva_python_semantic::current_python_version; @@ -137,8 +138,14 @@ fn run(f: impl FnOnce(Vec) -> Vec) -> anyhow::Resultconcise: Print diagnostics concisely, one per line
--quiet, -q

Use quiet output (or -qq for silent output)

--retry retry

When set, the test will retry failed tests up to this number of times

-
--snapshot-update

Update snapshots directly instead of creating pending .snap.new files.

+
--run-ignored run-ignored

Run ignored tests

+

Possible values:

+
    +
  • only: Run only ignored tests
    • +
  • all: Run both ignored and non-ignored tests
    • +
--snapshot-update

Update snapshots directly instead of creating pending .snap.new files.

When set, karva.assert_snapshot() will write directly to .snap files, accepting any changes automatically.

--test-prefix test-prefix

The prefix of the test functions

--try-import-fixtures

When set, we will try to import functions in each test file as well as parsing the ast to find them.