From 362832e2d0d5e54a985020062afabe5779ab74ef Mon Sep 17 00:00:00 2001 From: Jeremy Retailleau Date: Wed, 13 Aug 2025 18:23:41 -0700 Subject: [PATCH] Add TEST_PATHS option and improve pytest discovery error detection - Added `TEST_PATHS` to `pytest_discover_tests` to limit discovery to specific files or directories, defaulting to `testpaths` from pytest.ini if unset. - Updated error detection to catch both block-form and single-line messages from pytest. - Add tests. --- cmake/FindPytest.cmake | 45 +++++++-- cmake/PytestAddTests.cmake | 35 +++++-- doc/api_reference.rst | 14 +++ doc/release/release_notes.rst | 19 ++++ test/01-modify-name/CMakeLists.txt | 12 +-- test/07-working-directory/CMakeLists.txt | 18 ++++ .../subdir/test_correct.py | 2 + test/07-working-directory/test_incorrect.py | 1 + test/08-test-paths/CMakeLists.txt | 94 +++++++++++++++++++ test/08-test-paths/pytest.ini | 4 + test/08-test-paths/test_a/choice.py | 4 + test/08-test-paths/test_a/math/test_add.py | 2 + test/08-test-paths/test_a/test_upper.py | 2 + test/08-test-paths/test_b/math/test_power.py | 2 + .../test_b/math/test_subtract.py | 2 + test/08-test-paths/test_b/test_concat.py | 2 + test/08-test-paths/test_incorrect.py | 1 + test/CMakeLists.txt | 20 ++++ .../compare_discovered_tests.cmake | 0 19 files changed, 253 insertions(+), 26 deletions(-) create mode 100644 test/07-working-directory/CMakeLists.txt create mode 100644 test/07-working-directory/subdir/test_correct.py create mode 100644 test/07-working-directory/test_incorrect.py create mode 100644 test/08-test-paths/CMakeLists.txt create mode 100644 test/08-test-paths/pytest.ini create mode 100644 test/08-test-paths/test_a/choice.py create mode 100644 test/08-test-paths/test_a/math/test_add.py create mode 100644 test/08-test-paths/test_a/test_upper.py create mode 100644 test/08-test-paths/test_b/math/test_power.py create mode 100644 test/08-test-paths/test_b/math/test_subtract.py create mode 100644 test/08-test-paths/test_b/test_concat.py create mode 100644 test/08-test-paths/test_incorrect.py rename test/{01-modify-name => utils}/compare_discovered_tests.cmake (100%) diff --git a/cmake/FindPytest.cmake b/cmake/FindPytest.cmake index f3f12c1..963bf97 100644 --- a/cmake/FindPytest.cmake +++ b/cmake/FindPytest.cmake @@ -55,10 +55,34 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) # Function to discover pytest tests and add them to CTest. function(pytest_discover_tests NAME) + set(_BOOL_ARGS + STRIP_PARAM_BRACKETS + INCLUDE_FILE_PATH + BUNDLE_TESTS + ) + + set(_SINGLE_VALUE_ARGS + WORKING_DIRECTORY + TRIM_FROM_NAME + TRIM_FROM_FULL_NAME + ) + + set(_MULTI_VALUE_ARGS + TEST_PATHS + LIBRARY_PATH_PREPEND + PYTHON_PATH_PREPEND + ENVIRONMENT + PROPERTIES + DEPENDS + EXTRA_ARGS + DISCOVERY_EXTRA_ARGS + ) + cmake_parse_arguments( - PARSE_ARGV 1 "" "STRIP_PARAM_BRACKETS;INCLUDE_FILE_PATH;BUNDLE_TESTS" - "WORKING_DIRECTORY;TRIM_FROM_NAME;TRIM_FROM_FULL_NAME" - "LIBRARY_PATH_PREPEND;PYTHON_PATH_PREPEND;ENVIRONMENT;PROPERTIES;DEPENDS;EXTRA_ARGS;DISCOVERY_EXTRA_ARGS" + PARSE_ARGV 1 "" + "${_BOOL_ARGS}" + "${_SINGLE_VALUE_ARGS}" + "${_MULTI_VALUE_ARGS}" ) # Set platform-specific library path environment variable. @@ -119,6 +143,7 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) DEPENDS ${_DEPENDS} COMMAND ${CMAKE_COMMAND} -D "PYTEST_EXECUTABLE=${PYTEST_EXECUTABLE}" + -D "TEST_PATHS=${_TEST_PATHS}" -D "TEST_GROUP_NAME=${NAME}" -D "BUNDLE_TESTS=${_BUNDLE_TESTS}" -D "LIBRARY_ENV_NAME=${LIBRARY_ENV_NAME}" @@ -139,13 +164,13 @@ if (Pytest_FOUND AND NOT TARGET Pytest::Pytest) # Create a custom target to run the tests. add_custom_target(${NAME} ALL DEPENDS ${_tests_file}) - file(WRITE "${_include_file}" - "if(EXISTS \"${_tests_file}\")\n" - " include(\"${_tests_file}\")\n" - "else()\n" - " add_test(${NAME}_NOT_BUILT ${NAME}_NOT_BUILT)\n" - "endif()\n" - ) + file(WRITE "${_include_file}" + "if(EXISTS \"${_tests_file}\")\n" + " include(\"${_tests_file}\")\n" + "else()\n" + " add_test(${NAME}_NOT_BUILT ${NAME}_NOT_BUILT)\n" + "endif()\n" + ) # Register the include file to be processed for tests. set_property(DIRECTORY diff --git a/cmake/PytestAddTests.cmake b/cmake/PytestAddTests.cmake index 9542750..aa1eabe 100644 --- a/cmake/PytestAddTests.cmake +++ b/cmake/PytestAddTests.cmake @@ -38,14 +38,21 @@ if(CMAKE_SCRIPT_MODE_FILE) list(JOIN EXTRA_ARGS_WRAPPED " " EXTRA_ARGS_STR) # Macro to create individual tests with optional test properties. - macro(create_test NAME IDENTIFIER) - string(APPEND _content - "add_test([==[${NAME}]==] \"${PYTEST_EXECUTABLE}\" [==[${IDENTIFIER}]==] ${EXTRA_ARGS_STR} )\n" - ) + macro(create_test NAME IDENTIFIERS) + string(APPEND _content "add_test([==[${NAME}]==] \"${PYTEST_EXECUTABLE}\"") + + foreach(identifier ${IDENTIFIERS}) + string(APPEND _content " [==[${identifier}]==]") + endforeach() + + string(APPEND _content " ${EXTRA_ARGS_STR} )\n") # Prepare the properties for the test, including the environment settings. set(args "PROPERTIES ENVIRONMENT [==[${ENCODED_ENVIRONMENT}]==]") + # Add working directory + string(APPEND args " WORKING_DIRECTORY [==[${WORKING_DIRECTORY}]==]") + # Append any additional properties, escaping complex characters if necessary. foreach(property ${TEST_PROPERTIES}) if(property MATCHES "[^-./:a-zA-Z0-9_]") @@ -61,7 +68,7 @@ if(CMAKE_SCRIPT_MODE_FILE) # If tests are bundled together, create a single test group. if (BUNDLE_TESTS) - create_test("\${TEST_GROUP_NAME}" "\${WORKING_DIRECTORY}") + create_test("\${TEST_GROUP_NAME}" "\${TEST_PATHS}") else() # Set environment variables for collecting tests. @@ -69,11 +76,19 @@ if(CMAKE_SCRIPT_MODE_FILE) set(ENV{PYTHONPATH} "${PYTHON_PATH}") set(ENV{PYTHONWARNINGS} "ignore") + set(_command + "${PYTEST_EXECUTABLE}" --collect-only -q + "--rootdir=${WORKING_DIRECTORY}" + ${DISCOVERY_EXTRA_ARGS} + ) + + foreach(test_path IN LISTS TEST_PATHS) + list(APPEND _command "${test_path}") + endforeach() + # Collect tests. execute_process( - COMMAND "${PYTEST_EXECUTABLE}" - --collect-only -q - --rootdir=${WORKING_DIRECTORY} ${DISCOVERY_EXTRA_ARGS} . + COMMAND ${_command} OUTPUT_VARIABLE _output_lines ERROR_VARIABLE _output_lines OUTPUT_STRIP_TRAILING_WHITESPACE @@ -81,7 +96,7 @@ if(CMAKE_SCRIPT_MODE_FILE) ) # Check for errors during test collection. - string(REGEX MATCH "=+ ERRORS =+(.*)" _error "${_output_lines}") + string(REGEX MATCH "(=+ ERRORS =+|ERROR:).*" _error "${_output_lines}") if (_error) message(${_error}) @@ -141,7 +156,7 @@ if(CMAKE_SCRIPT_MODE_FILE) # Prefix the test name with the test group name. set(test_name "${TEST_GROUP_NAME}.${test_name}") - set(test_case "${WORKING_DIRECTORY}/${line}") + set(test_case "${line}") # Create the test for CTest. create_test("\${test_name}" "\${test_case}") diff --git a/doc/api_reference.rst b/doc/api_reference.rst index 712e8ea..d24d883 100644 --- a/doc/api_reference.rst +++ b/doc/api_reference.rst @@ -12,6 +12,7 @@ API Reference with :term:`Pytest` within a controlled environment:: pytest_discover_tests(NAME + [TEST_PATHS path1 path2...] [WORKING_DIRECTORY dir] [TRIM_FROM_NAME pattern] [TRIM_FROM_FULL_NAME pattern] @@ -35,6 +36,19 @@ API Reference used as a prefix for each test created, or as an identifier the bundled test. + * ``TEST_PATHS`` + + Specifies a list of files or directories to search when executing + :term:`Pytest` from the current source directory (or from the + ``WORKING_DIRECTORY`` value if provided):: + + pytest_discover_tests( + ... + TEST_PATHS + path1 + path2/test.py + ) + * ``WORKING_DIRECTORY`` Specify the directory in which to run the :term:`Pytest` command. If diff --git a/doc/release/release_notes.rst b/doc/release/release_notes.rst index 4286e71..c5d322c 100644 --- a/doc/release/release_notes.rst +++ b/doc/release/release_notes.rst @@ -10,6 +10,25 @@ Release Notes Added compatibility with CMake 4.1. + .. change:: new + + Added ``TEST_PATHS`` option to the :func:`pytest_discover_tests` + function, allowing users to limit test discovery and execution to + specific files or directories. If not specified, ``TEST_PATHS`` + defaults to the ``testpaths`` setting in :file:`pytest.ini`, or to the + current directory if ``testpaths`` is not set. This matches + :term:`Pytest`’s native behavior and preserves full backward + compatibility. + + .. seealso:: + + `'testpaths' configuration option + `_ + + .. change:: fixed + + Updated discovery error detection to recognize both block-form and + single-line messages from :term:`Pytest`. .. release:: 0.13.0 :date: 2025-02-16 diff --git a/test/01-modify-name/CMakeLists.txt b/test/01-modify-name/CMakeLists.txt index a440ca8..55ec950 100644 --- a/test/01-modify-name/CMakeLists.txt +++ b/test/01-modify-name/CMakeLists.txt @@ -28,7 +28,7 @@ add_test(NAME TestModifyName.Validate.Simple COMMAND ${CMAKE_COMMAND} -D "TEST_PREFIX=TestModifyName.Simple" -D "EXPECTED=${EXPECTED}" - -P ${CMAKE_CURRENT_LIST_DIR}/compare_discovered_tests.cmake + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake ) pytest_discover_tests(TestModifyName.Bundled BUNDLE_TESTS) @@ -36,7 +36,7 @@ add_test(NAME TestModifyName.Validate.Bundled COMMAND ${CMAKE_COMMAND} -D "TEST_PREFIX=TestModifyName.Bundled" -D "EXPECTED=TestModifyName.Bundled" - -P ${CMAKE_CURRENT_LIST_DIR}/compare_discovered_tests.cmake + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake ) pytest_discover_tests( @@ -64,7 +64,7 @@ add_test(NAME TestModifyName.Validate.TrimFromName COMMAND ${CMAKE_COMMAND} -D "TEST_PREFIX=TestModifyName.TrimFromName" -D "EXPECTED=${EXPECTED}" - -P ${CMAKE_CURRENT_LIST_DIR}/compare_discovered_tests.cmake + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake ) pytest_discover_tests( @@ -92,7 +92,7 @@ add_test(NAME TestModifyName.Validate.StripParamBrackets COMMAND ${CMAKE_COMMAND} -D "TEST_PREFIX=TestModifyName.StripParamBrackets" -D "EXPECTED=${EXPECTED}" - -P ${CMAKE_CURRENT_LIST_DIR}/compare_discovered_tests.cmake + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake ) pytest_discover_tests( @@ -120,7 +120,7 @@ add_test(NAME TestModifyName.Validate.IncludeFilePath COMMAND ${CMAKE_COMMAND} -D "TEST_PREFIX=TestModifyName.IncludeFilePath" -D "EXPECTED=${EXPECTED}" - -P ${CMAKE_CURRENT_LIST_DIR}/compare_discovered_tests.cmake + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake ) pytest_discover_tests( @@ -149,5 +149,5 @@ add_test(NAME TestModifyName.Validate.TrimFromFullName COMMAND ${CMAKE_COMMAND} -D "TEST_PREFIX=TestModifyName.TrimFromFullName" -D "EXPECTED=${EXPECTED}" - -P ${CMAKE_CURRENT_LIST_DIR}/compare_discovered_tests.cmake + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake ) diff --git a/test/07-working-directory/CMakeLists.txt b/test/07-working-directory/CMakeLists.txt new file mode 100644 index 0000000..f52b775 --- /dev/null +++ b/test/07-working-directory/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.20) + +project(TestWorkingDirectory) + +find_package(Pytest REQUIRED) + +enable_testing() + +pytest_discover_tests( + TestWorkingDirectory + WORKING_DIRECTORY subdir +) + +pytest_discover_tests( + TestWorkingDirectory.Bundled + WORKING_DIRECTORY subdir + BUNDLE_TESTS +) diff --git a/test/07-working-directory/subdir/test_correct.py b/test/07-working-directory/subdir/test_correct.py new file mode 100644 index 0000000..ca4655d --- /dev/null +++ b/test/07-working-directory/subdir/test_correct.py @@ -0,0 +1,2 @@ +def test_addition(): + assert 1 + 1 == 2 diff --git a/test/07-working-directory/test_incorrect.py b/test/07-working-directory/test_incorrect.py new file mode 100644 index 0000000..fffaf92 --- /dev/null +++ b/test/07-working-directory/test_incorrect.py @@ -0,0 +1 @@ +raise RuntimeError("This test files should have been excluded") diff --git a/test/08-test-paths/CMakeLists.txt b/test/08-test-paths/CMakeLists.txt new file mode 100644 index 0000000..8ebb3c7 --- /dev/null +++ b/test/08-test-paths/CMakeLists.txt @@ -0,0 +1,94 @@ +cmake_minimum_required(VERSION 3.20) + +project(TestCustomPaths) + +find_package(Pytest REQUIRED) + +enable_testing() + +pytest_discover_tests( + TestCustomPaths.FromConfig +) +set(EXPECTED + "TestCustomPaths.FromConfig.test_addition" + "TestCustomPaths.FromConfig.test_concat" + "TestCustomPaths.FromConfig.test_power" + "TestCustomPaths.FromConfig.test_substraction" + "TestCustomPaths.FromConfig.test_upper" +) +add_test(NAME TestCustomPaths.Validate.FromConfig + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestCustomPaths.FromConfig" + -D "EXPECTED=${EXPECTED}" + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake +) + +pytest_discover_tests( + TestCustomPaths.BundledFromConfig + BUNDLE_TESTS +) +set(EXPECTED + "TestCustomPaths.BundledFromConfig" +) +add_test(NAME TestCustomPaths.Validate.BundledFromConfig + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestCustomPaths.BundledFromConfig" + -D "EXPECTED=${EXPECTED}" + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake +) + +pytest_discover_tests( + TestCustomPaths.FilePaths + TEST_PATHS + ${CMAKE_CURRENT_LIST_DIR}/test_a/math/test_add.py + test_a/choice.py + test_b/test_concat.py +) +set(EXPECTED + "TestCustomPaths.FilePaths.test_addition" + "TestCustomPaths.FilePaths.test_concat" + "TestCustomPaths.FilePaths.test_random" +) +add_test(NAME TestCustomPaths.Validate.FilePaths + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestCustomPaths.FilePaths" + -D "EXPECTED=${EXPECTED}" + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake +) + +pytest_discover_tests( + TestCustomPaths.DirPaths + TEST_PATHS + test_a + ${CMAKE_CURRENT_LIST_DIR}/test_b/math +) +set(EXPECTED + "TestCustomPaths.DirPaths.test_addition" + "TestCustomPaths.DirPaths.test_power" + "TestCustomPaths.DirPaths.test_substraction" + "TestCustomPaths.DirPaths.test_upper" +) +add_test(NAME TestCustomPaths.Validate.DirPaths + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestCustomPaths.DirPaths" + -D "EXPECTED=${EXPECTED}" + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake +) + +pytest_discover_tests( + TestCustomPaths.WithWorkingDirectory + TEST_PATHS + math + choice.py + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/test_a +) +set(EXPECTED + "TestCustomPaths.WithWorkingDirectory.test_addition" + "TestCustomPaths.WithWorkingDirectory.test_random" +) +add_test(NAME TestCustomPaths.Validate.WithWorkingDirectory + COMMAND ${CMAKE_COMMAND} + -D "TEST_PREFIX=TestCustomPaths.WithWorkingDirectory" + -D "EXPECTED=${EXPECTED}" + -P ${CMAKE_CURRENT_LIST_DIR}/../utils/compare_discovered_tests.cmake +) diff --git a/test/08-test-paths/pytest.ini b/test/08-test-paths/pytest.ini new file mode 100644 index 0000000..d40fc3d --- /dev/null +++ b/test/08-test-paths/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +testpaths = + test_a + test_b diff --git a/test/08-test-paths/test_a/choice.py b/test/08-test-paths/test_a/choice.py new file mode 100644 index 0000000..5260b87 --- /dev/null +++ b/test/08-test-paths/test_a/choice.py @@ -0,0 +1,4 @@ +import random + +def test_random(): + assert random.choice([1, 2, 3]) in (1, 2, 3) diff --git a/test/08-test-paths/test_a/math/test_add.py b/test/08-test-paths/test_a/math/test_add.py new file mode 100644 index 0000000..ca4655d --- /dev/null +++ b/test/08-test-paths/test_a/math/test_add.py @@ -0,0 +1,2 @@ +def test_addition(): + assert 1 + 1 == 2 diff --git a/test/08-test-paths/test_a/test_upper.py b/test/08-test-paths/test_a/test_upper.py new file mode 100644 index 0000000..0cbab20 --- /dev/null +++ b/test/08-test-paths/test_a/test_upper.py @@ -0,0 +1,2 @@ +def test_upper(): + assert "hello".upper() == "HELLO" diff --git a/test/08-test-paths/test_b/math/test_power.py b/test/08-test-paths/test_b/math/test_power.py new file mode 100644 index 0000000..e93fef6 --- /dev/null +++ b/test/08-test-paths/test_b/math/test_power.py @@ -0,0 +1,2 @@ +def test_power(): + assert 2 ** 3 == 8 diff --git a/test/08-test-paths/test_b/math/test_subtract.py b/test/08-test-paths/test_b/math/test_subtract.py new file mode 100644 index 0000000..ce05878 --- /dev/null +++ b/test/08-test-paths/test_b/math/test_subtract.py @@ -0,0 +1,2 @@ +def test_substraction(): + assert 2 - 1 == 1 diff --git a/test/08-test-paths/test_b/test_concat.py b/test/08-test-paths/test_b/test_concat.py new file mode 100644 index 0000000..2186c84 --- /dev/null +++ b/test/08-test-paths/test_b/test_concat.py @@ -0,0 +1,2 @@ +def test_concat(): + assert "foo" + "bar" == "foobar" diff --git a/test/08-test-paths/test_incorrect.py b/test/08-test-paths/test_incorrect.py new file mode 100644 index 0000000..fffaf92 --- /dev/null +++ b/test/08-test-paths/test_incorrect.py @@ -0,0 +1 @@ +raise RuntimeError("This test files should have been excluded") diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6eef290..c53f224 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -62,3 +62,23 @@ ExternalProject_Add( TEST_COMMAND ${CMAKE_CTEST_COMMAND} -C Release -VV ) + +ExternalProject_Add( + TestWorkingDirectory + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/07-working-directory + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps/07-working-directory + BUILD_COMMAND ${CMAKE_COMMAND} --build + INSTALL_COMMAND "" + TEST_COMMAND ${CMAKE_CTEST_COMMAND} + -C Release -VV +) + +ExternalProject_Add( + TestCustomPaths + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/08-test-paths + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps/08-test-paths + BUILD_COMMAND ${CMAKE_COMMAND} --build + INSTALL_COMMAND "" + TEST_COMMAND ${CMAKE_CTEST_COMMAND} + -C Release -VV +) diff --git a/test/01-modify-name/compare_discovered_tests.cmake b/test/utils/compare_discovered_tests.cmake similarity index 100% rename from test/01-modify-name/compare_discovered_tests.cmake rename to test/utils/compare_discovered_tests.cmake