diff --git a/Cargo.lock b/Cargo.lock index a5c4345d..87360f89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1356,6 +1356,7 @@ dependencies = [ "karva_combine", "karva_macros", "karva_system", + "regex", "ruff_db", "ruff_options_metadata", "ruff_python_ast", diff --git a/crates/karva/src/lib.rs b/crates/karva/src/lib.rs index 346b79ca..3a1747c1 100644 --- a/crates/karva/src/lib.rs +++ b/crates/karva/src/lib.rs @@ -11,7 +11,7 @@ use colored::Colorize; use karva_cache::AggregatedResults; use karva_cli::{Args, Command, OutputFormat, TestCommand}; use karva_logging::{Printer, set_colored_override, setup_tracing}; -use karva_metadata::filter::TagFilterSet; +use karva_metadata::filter::{NameFilterSet, TagFilterSet}; use karva_metadata::{ProjectMetadata, ProjectOptionsOverrides}; use karva_project::ProjectDatabase; use karva_python_semantic::current_python_version; @@ -119,6 +119,7 @@ pub(crate) fn test(args: TestCommand) -> Result { }; TagFilterSet::new(&sub_command.tag_expressions)?; + NameFilterSet::new(&sub_command.name_patterns)?; let config = karva_runner::ParallelTestConfig { num_workers, diff --git a/crates/karva/tests/it/main.rs b/crates/karva/tests/it/main.rs index 4b36ff3e..fe646336 100644 --- a/crates/karva/tests/it/main.rs +++ b/crates/karva/tests/it/main.rs @@ -4,3 +4,4 @@ mod basic; mod configuration; mod discovery; mod extensions; +mod name_filter; diff --git a/crates/karva/tests/it/name_filter.rs b/crates/karva/tests/it/name_filter.rs new file mode 100644 index 00000000..f997ec7f --- /dev/null +++ b/crates/karva/tests/it/name_filter.rs @@ -0,0 +1,427 @@ +use insta_cmd::assert_cmd_snapshot; + +use crate::common::TestContext; + +const TWO_TESTS: &str = r" +def test_alpha(): + assert True + +def test_beta(): + assert True +"; + +#[test] +fn name_filter_substring_match() { + let context = TestContext::with_file("test.py", TWO_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("alpha"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_alpha ... ok + test test::test_beta ... skipped + + test result: ok. 1 passed; 0 failed; 1 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_anchored_regex() { + let context = TestContext::with_file("test.py", TWO_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("beta$"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_alpha ... skipped + test test::test_beta ... ok + + test result: ok. 1 passed; 0 failed; 1 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_multiple_flags_or_semantics() { + let context = TestContext::with_file("test.py", TWO_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("alpha").arg("-m").arg("beta"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_alpha ... ok + test test::test_beta ... ok + + test result: ok. 2 passed; 0 failed; 0 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_no_matches() { + let context = TestContext::with_file("test.py", TWO_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("nonexistent"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_alpha ... skipped + test test::test_beta ... skipped + + test result: ok. 0 passed; 0 failed; 2 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_invalid_regex() { + let context = TestContext::with_file("test.py", TWO_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("[invalid"), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Karva failed + Cause: invalid regex pattern `[invalid`: regex parse error: + [invalid + ^ + error: unclosed character class + "); +} + +#[test] +fn name_filter_parametrize() { + let context = TestContext::with_file( + "test.py", + r" +import karva + +@karva.tags.parametrize('x', [1, 2, 3]) +def test_param(x): + assert x > 0 + +def test_other(): + assert True + ", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("test_param"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_param(x=1) ... ok + test test::test_param(x=2) ... ok + test test::test_param(x=3) ... ok + test test::test_other ... skipped + + test result: ok. 3 passed; 0 failed; 1 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +#[cfg(unix)] +fn name_filter_match_all() { + let context = TestContext::with_file("test.py", TWO_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg(".*"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_alpha ... ok + test test::test_beta ... ok + + test result: ok. 2 passed; 0 failed; 0 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_alternation() { + let context = TestContext::with_file( + "test.py", + r" +def test_login(): + assert True + +def test_logout(): + assert True + +def test_signup(): + assert True + ", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("login|signup"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_login ... ok + test test::test_logout ... skipped + test test::test_signup ... ok + + test result: ok. 2 passed; 0 failed; 1 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_character_class() { + let context = TestContext::with_file( + "test.py", + r" +def test_v1(): + assert True + +def test_v2(): + assert True + +def test_v10(): + assert True + ", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg(r"test_v[12]$"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_v1 ... ok + test test::test_v2 ... ok + test test::test_v10 ... skipped + + test result: ok. 2 passed; 0 failed; 1 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_quantifier() { + let context = TestContext::with_file( + "test.py", + r" +def test_a(): + assert True + +def test_ab(): + assert True + +def test_abb(): + assert True + ", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg(r"test_ab+$"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_a ... skipped + test test::test_ab ... ok + test test::test_abb ... ok + + test result: ok. 2 passed; 0 failed; 1 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_qualified_name_prefix() { + let context = TestContext::with_file( + "test.py", + r" +def test_login(): + assert True + +def test_logout(): + assert True + +def test_signup(): + assert True + ", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("^test::test_log"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_login ... ok + test test::test_logout ... ok + test test::test_signup ... skipped + + test result: ok. 2 passed; 0 failed; 1 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_combined_with_tag_filter() { + let context = TestContext::with_file( + "test.py", + r" +import karva + +@karva.tags.slow +def test_slow_alpha(): + assert True + +@karva.tags.slow +def test_slow_beta(): + assert True + +def test_fast_alpha(): + assert True + ", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("-t").arg("slow").arg("-m").arg("alpha"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_slow_alpha ... ok + test test::test_slow_beta ... skipped + test test::test_fast_alpha ... skipped + + test result: ok. 1 passed; 0 failed; 2 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_case_sensitive() { + let context = TestContext::with_file( + "test.py", + r" +def test_Alpha(): + assert True + +def test_alpha(): + assert True + ", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("Alpha"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_Alpha ... ok + test test::test_alpha ... skipped + + test result: ok. 1 passed; 0 failed; 1 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_case_insensitive_regex() { + let context = TestContext::with_file( + "test.py", + r" +def test_Alpha(): + assert True + +def test_alpha(): + assert True + ", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("(?i)alpha"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_Alpha ... ok + test test::test_alpha ... ok + + test result: ok. 2 passed; 0 failed; 0 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_dot_metacharacter() { + let context = TestContext::with_file( + "test.py", + r" +def test_a1(): + assert True + +def test_a2(): + assert True + +def test_ab(): + assert True + ", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg(r"test_a\d"), @r" + success: true + exit_code: 0 + ----- stdout ----- + test test::test_a1 ... ok + test test::test_a2 ... ok + test test::test_ab ... skipped + + test result: ok. 2 passed; 0 failed; 1 skipped; finished in [TIME] + + ----- stderr ----- + "); +} + +#[test] +fn name_filter_invalid_regex_unclosed_group() { + let context = TestContext::with_file("test.py", TWO_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("(unclosed"), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Karva failed + Cause: invalid regex pattern `(unclosed`: regex parse error: + (unclosed + ^ + error: unclosed group + "); +} + +#[test] +fn name_filter_invalid_regex_invalid_repetition() { + let context = TestContext::with_file("test.py", TWO_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg("*invalid"), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Karva failed + Cause: invalid regex pattern `*invalid`: regex parse error: + *invalid + ^ + error: repetition operator missing expression + "); +} + +#[test] +fn name_filter_invalid_regex_bad_escape() { + let context = TestContext::with_file("test.py", TWO_TESTS); + assert_cmd_snapshot!(context.command_no_parallel().arg("-m").arg(r"\p{Invalid}"), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Karva failed + Cause: invalid regex pattern `\p{Invalid}`: regex parse error: + \p{Invalid} + ^^^^^^^^^^^ + error: Unicode property not found + "); +} diff --git a/crates/karva_cli/src/lib.rs b/crates/karva_cli/src/lib.rs index 97786379..6955b890 100644 --- a/crates/karva_cli/src/lib.rs +++ b/crates/karva_cli/src/lib.rs @@ -146,6 +146,16 @@ pub struct SubTestCommand { #[clap(short = 't', long = "tag")] pub tag_expressions: Vec, + /// Filter tests by name using a regular expression. + /// + /// Only tests whose fully qualified name matches the pattern will run. + /// Uses partial matching (the pattern can match anywhere in the name). + /// When specified multiple times, a test runs if it matches any of the patterns. + /// + /// Examples: `-m auth`, `-m '^test::test_login'`, `-m 'slow|fast'`. + #[clap(short = 'm', long = "match")] + pub name_patterns: Vec, + #[clap(flatten)] pub verbosity: Verbosity, } diff --git a/crates/karva_core/src/cli.rs b/crates/karva_core/src/cli.rs index 01dc3e07..5900a8c2 100644 --- a/crates/karva_core/src/cli.rs +++ b/crates/karva_core/src/cli.rs @@ -10,7 +10,7 @@ use karva_cache::{Cache, RunHash}; use karva_cli::{SubTestCommand, Verbosity}; use karva_diagnostic::{DummyReporter, Reporter, TestCaseReporter}; use karva_logging::{Printer, set_colored_override, setup_tracing}; -use karva_metadata::filter::TagFilterSet; +use karva_metadata::filter::{NameFilterSet, TagFilterSet}; use karva_python_semantic::current_python_version; use karva_system::System; use karva_system::path::{TestPath, TestPathError}; @@ -143,11 +143,13 @@ fn run(f: impl FnOnce(Vec) -> Vec) -> anyhow::Result PackageRunner<'ctx, 'a> { } } + let name_filter = &self.context.settings().test().name_filter; + if !name_filter.is_empty() { + let display_name = QualifiedTestName::new(name.clone(), None).to_string(); + if !name_filter.matches(&display_name) { + return self.context.register_test_case_result( + &QualifiedTestName::new(name, None), + IndividualTestResultKind::Skipped { reason: None }, + std::time::Duration::ZERO, + ); + } + } + if let (true, reason) = tags.should_skip() { return self.context.register_test_case_result( &QualifiedTestName::new(name, None), diff --git a/crates/karva_metadata/Cargo.toml b/crates/karva_metadata/Cargo.toml index 90529f04..23324eeb 100644 --- a/crates/karva_metadata/Cargo.toml +++ b/crates/karva_metadata/Cargo.toml @@ -18,6 +18,7 @@ karva_macros = { workspace = true } karva_system = { workspace = true } camino = { workspace = true } +regex = { workspace = true } ruff_db = { workspace = true } ruff_options_metadata = { workspace = true } ruff_python_ast = { workspace = true } diff --git a/crates/karva_metadata/src/filter.rs b/crates/karva_metadata/src/filter.rs index cf0e2e3c..7e6470ad 100644 --- a/crates/karva_metadata/src/filter.rs +++ b/crates/karva_metadata/src/filter.rs @@ -1,5 +1,72 @@ use std::fmt; +use regex::Regex; + +/// A name filter that matches test names using a regular expression. +#[derive(Debug, Clone)] +pub struct NameFilter { + regex: Regex, +} + +impl NameFilter { + pub fn new(pattern: &str) -> Result { + let regex = Regex::new(pattern).map_err(|err| NameFilterError::InvalidRegex { + pattern: pattern.to_string(), + source: err, + })?; + Ok(Self { regex }) + } + + pub fn matches(&self, name: &str) -> bool { + self.regex.is_match(name) + } +} + +/// A set of name filters. Any filter must match for the set to match (OR semantics across `-m` flags). +#[derive(Debug, Clone, Default)] +pub struct NameFilterSet { + filters: Vec, +} + +impl NameFilterSet { + pub fn new(patterns: &[String]) -> Result { + let mut filters = Vec::with_capacity(patterns.len()); + for pattern in patterns { + filters.push(NameFilter::new(pattern)?); + } + Ok(Self { filters }) + } + + pub fn is_empty(&self) -> bool { + self.filters.is_empty() + } + + pub fn matches(&self, name: &str) -> bool { + self.filters.is_empty() || self.filters.iter().any(|f| f.matches(name)) + } +} + +/// Error that occurs when parsing a name filter pattern. +#[derive(Debug)] +pub enum NameFilterError { + InvalidRegex { + pattern: String, + source: regex::Error, + }, +} + +impl fmt::Display for NameFilterError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidRegex { pattern, source } => { + write!(f, "invalid regex pattern `{pattern}`: {source}") + } + } + } +} + +impl std::error::Error for NameFilterError {} + /// A parsed tag filter expression that can be matched against a set of tag names. #[derive(Debug, Clone)] pub struct TagFilter { @@ -668,4 +735,70 @@ mod tests { fn filter_set_rejects_invalid_expression() { assert!(TagFilterSet::new(&["slow".to_string(), "and".to_string()]).is_err()); } + + #[test] + fn name_filter_partial_match() { + let f = NameFilter::new("auth").expect("parse"); + assert!(f.matches("test::test_auth_login")); + assert!(f.matches("test::test_auth")); + assert!(!f.matches("test::test_login")); + } + + #[test] + fn name_filter_anchored_start() { + let f = NameFilter::new("^test::test_login").expect("parse"); + assert!(f.matches("test::test_login")); + assert!(f.matches("test::test_login_flow")); + assert!(!f.matches("other::test_login")); + } + + #[test] + fn name_filter_anchored_end() { + let f = NameFilter::new("login$").expect("parse"); + assert!(f.matches("test::test_login")); + assert!(!f.matches("test::test_login_flow")); + } + + #[test] + fn name_filter_alternation() { + let f = NameFilter::new("slow|fast").expect("parse"); + assert!(f.matches("test::test_slow")); + assert!(f.matches("test::test_fast")); + assert!(!f.matches("test::test_medium")); + } + + #[test] + fn name_filter_invalid_regex() { + assert!(matches!( + NameFilter::new("[invalid"), + Err(NameFilterError::InvalidRegex { .. }) + )); + } + + #[test] + fn name_filter_set_or_semantics() { + let set = NameFilterSet::new(&["test_a".to_string(), "test_b".to_string()]).expect("parse"); + assert!(set.matches("test::test_a")); + assert!(set.matches("test::test_b")); + assert!(!set.matches("test::test_c")); + } + + #[test] + fn name_filter_set_empty_always_matches() { + let set = NameFilterSet::new(&[]).expect("parse"); + assert!(set.is_empty()); + assert!(set.matches("anything")); + } + + #[test] + fn name_filter_set_rejects_invalid_pattern() { + assert!(NameFilterSet::new(&["valid".to_string(), "[invalid".to_string()]).is_err()); + } + + #[test] + fn name_filter_parametrized_name() { + let f = NameFilter::new(r"param=1").expect("parse"); + assert!(f.matches("test::test_add(param=1)")); + assert!(!f.matches("test::test_add(param=2)")); + } } diff --git a/crates/karva_metadata/src/options.rs b/crates/karva_metadata/src/options.rs index df4cbf0c..20d8551c 100644 --- a/crates/karva_metadata/src/options.rs +++ b/crates/karva_metadata/src/options.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::System; -use crate::filter::TagFilterSet; +use crate::filter::{NameFilterSet, TagFilterSet}; use crate::settings::{ProjectSettings, SrcSettings, TerminalSettings, TestSettings}; #[derive( @@ -219,6 +219,7 @@ impl TestOptions { try_import_fixtures: self.try_import_fixtures.unwrap_or_default(), retry: self.retry.unwrap_or_default(), tag_filter: TagFilterSet::default(), + name_filter: NameFilterSet::default(), } } } diff --git a/crates/karva_metadata/src/settings.rs b/crates/karva_metadata/src/settings.rs index 7eacf9f0..02a2d0bb 100644 --- a/crates/karva_metadata/src/settings.rs +++ b/crates/karva_metadata/src/settings.rs @@ -1,4 +1,4 @@ -use crate::filter::TagFilterSet; +use crate::filter::{NameFilterSet, TagFilterSet}; use crate::options::OutputFormat; #[derive(Default, Debug, Clone)] @@ -28,6 +28,10 @@ impl ProjectSettings { pub fn set_tag_filter(&mut self, tag_filter: TagFilterSet) { self.test.tag_filter = tag_filter; } + + pub fn set_name_filter(&mut self, name_filter: NameFilterSet) { + self.test.name_filter = name_filter; + } } #[derive(Default, Debug, Clone)] @@ -49,4 +53,5 @@ pub struct TestSettings { pub try_import_fixtures: bool, pub retry: u32, pub tag_filter: TagFilterSet, + pub name_filter: NameFilterSet, } diff --git a/crates/karva_runner/src/orchestration.rs b/crates/karva_runner/src/orchestration.rs index 8226ddc4..083d3c8c 100644 --- a/crates/karva_runner/src/orchestration.rs +++ b/crates/karva_runner/src/orchestration.rs @@ -319,6 +319,7 @@ fn inner_cli_args(settings: &ProjectSettings, args: &SubTestCommand) -> Vec = args.tag_expressions.clone(); + let name_patterns: Vec = args.name_patterns.clone(); cli_args .iter() @@ -328,5 +329,10 @@ fn inner_cli_args(settings: &ProjectSettings, args: &SubTestCommand) -> VecMay also be set with the KARVA_CONFIG_FILE environment variable.

--fail-fast

When set, the test will fail immediately if any test fails.

This only works when running tests in parallel.

--help, -h

Print help (see a summary with '-h')

+
--match, -m name-patterns

Filter tests by name using a regular expression.

+

Only tests whose fully qualified name matches the pattern will run. Uses partial matching (the pattern can match anywhere in the name). When specified multiple times, a test runs if it matches any of the patterns.

+

Examples: -m auth, -m '^test::test_login', -m 'slow|fast'.

--no-cache

Disable reading the karva cache for test duration history

--no-ignore

When set, .gitignore files will not be respected

--no-parallel

Disable parallel execution (equivalent to --num-workers 1)