From c15c563864c6aededef2e3d77d532570f3757cd2 Mon Sep 17 00:00:00 2001 From: NgoQuocViet2001 Date: Sun, 12 Jul 2026 01:48:51 +0700 Subject: [PATCH] Fix relative PRE_TEST discovery paths Closes #3051 --- docs/cmake-integration.md | 7 ++ extras/Catch.cmake | 91 ++++++++++++++--- .../TestScripts/DiscoverTests/CMakeLists.txt | 16 +-- .../DiscoverTests/VerifyRegistration.py | 98 +++++++++++++++---- .../relative-registration/CMakeLists.txt | 8 ++ 5 files changed, 181 insertions(+), 39 deletions(-) create mode 100644 tests/TestScripts/DiscoverTests/relative-registration/CMakeLists.txt diff --git a/docs/cmake-integration.md b/docs/cmake-integration.md index 63b5ebdedf..14a68bec2b 100644 --- a/docs/cmake-integration.md +++ b/docs/cmake-integration.md @@ -127,6 +127,7 @@ catch_discover_tests(target [OUTPUT_PREFIX prefix] [OUTPUT_SUFFIX suffix] [DISCOVERY_MODE ] + [USE_RELATIVE_PATHS] [SKIP_IS_FAILURE] [ADD_TAGS_AS_LABELS] ) @@ -223,6 +224,12 @@ limitation affects CMake's `gtest_discover_tests`; see [Catch2 #2411](https://github.com/catchorg/Catch2/issues/2411) and [CMake #21845](https://gitlab.kitware.com/cmake/cmake/-/issues/21845)._ +* `USE_RELATIVE_PATHS` + +With `DISCOVERY_MODE PRE_TEST`, stores generated discovery paths relative to +the CTest include file. This allows the build tree and test executable to be +relocated before test discovery runs. This option requires CMake 3.24 or newer. + * `SKIP_IS_FAILURE` Skipped tests will be marked as failed instead. diff --git a/extras/Catch.cmake b/extras/Catch.cmake index c143d330b0..e998972e62 100644 --- a/extras/Catch.cmake +++ b/extras/Catch.cmake @@ -38,6 +38,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. [OUTPUT_PREFIX prefix] [OUTPUT_SUFFIX suffix] [DISCOVERY_MODE ] + [USE_RELATIVE_PATHS] [SKIP_IS_FAILURE] [ADD_TAGS_AS_LABELS] ) @@ -157,6 +158,12 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. code-signs the test executable only after the post-build script that ``POST_BUILD`` mode uses to run it for test discovery. See Catch2 issue #2411. + ``USE_RELATIVE_PATHS`` + Makes the files generated for ``PRE_TEST`` discovery use paths relative to + the generated CTest include file. This allows a build tree to be relocated + with its test executable before test discovery runs. This option requires + CMake 3.24 or newer. + ``SKIP_IS_FAILURE`` Disables skipped test detection. @@ -170,7 +177,7 @@ function(catch_discover_tests TARGET) cmake_parse_arguments( "" - "SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS" + "SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS;USE_RELATIVE_PATHS" "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX;DISCOVERY_MODE" "TEST_SPEC;EXTRA_ARGS;PROPERTIES;DL_PATHS;DL_FRAMEWORK_PATHS" ${ARGN} @@ -201,6 +208,12 @@ function(catch_discover_tests TARGET) if(NOT _DISCOVERY_MODE MATCHES "^(POST_BUILD|PRE_TEST)$") message(FATAL_ERROR "Unknown DISCOVERY_MODE: ${_DISCOVERY_MODE}") endif() + if(_USE_RELATIVE_PATHS AND NOT _DISCOVERY_MODE STREQUAL "PRE_TEST") + message(FATAL_ERROR "USE_RELATIVE_PATHS requires DISCOVERY_MODE PRE_TEST") + endif() + if(_USE_RELATIVE_PATHS AND CMAKE_VERSION VERSION_LESS "3.24") + message(FATAL_ERROR "USE_RELATIVE_PATHS requires CMake 3.24 or newer") + endif() ## Generate a unique name based on the extra arguments string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX}") @@ -210,6 +223,7 @@ function(catch_discover_tests TARGET) set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-${args_hash}") set(ctest_include_file "${ctest_file_base}_include.cmake") set(ctest_tests_file "${ctest_file_base}_tests.cmake") + set(ctest_include_file_for_property "${ctest_include_file}") get_property(crosscompiling_emulator TARGET ${TARGET} @@ -264,16 +278,69 @@ function(catch_discover_tests TARGET) set(ctest_tests_file "${ctest_file_base}_tests-$.cmake") endif() + set(test_executable_for_script "$") + set(test_working_dir_for_script "${_WORKING_DIRECTORY}") + set(discover_tests_script_for_script "${_CATCH_DISCOVER_TESTS_SCRIPT}") + set(ctest_tests_file_for_script "${ctest_tests_file}") + set(test_executable_argument "[==[${test_executable_for_script}]==]") + set(test_working_dir_argument "[==[${test_working_dir_for_script}]==]") + set(ctest_file_argument "[==[${ctest_tests_file_for_script}]==]") + set(ctest_path_setup) + set(ctest_config_include_file + "${ctest_file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake" + ) + + if(_USE_RELATIVE_PATHS) + get_filename_component(ctest_file_base_name "${ctest_file_base}" NAME) + get_filename_component(ctest_tests_file_name "${ctest_tests_file}" NAME) + get_filename_component(ctest_include_file_name "${ctest_include_file}" NAME) + + set(test_executable_relative + "$,${CMAKE_CURRENT_BINARY_DIR}>" + ) + set(test_working_dir_relative + "$,${CMAKE_CURRENT_BINARY_DIR}>" + ) + set(discover_tests_script_name + "${ctest_file_base_name}_CatchAddTests.cmake" + ) + configure_file( + "${_CATCH_DISCOVER_TESTS_SCRIPT}" + "${CMAKE_CURRENT_BINARY_DIR}/${discover_tests_script_name}" + COPYONLY + ) + + set(ctest_config_include_file + "${ctest_file_base_name}_include-\${CTEST_CONFIGURATION_TYPE}.cmake" + ) + string(CONCAT ctest_path_setup + "get_filename_component(_catch_discover_tests_script \"${discover_tests_script_name}\" REALPATH)" "\n" + "get_filename_component(_catch_discovery_dir \"\${_catch_discover_tests_script}\" DIRECTORY)" "\n" + "set(_catch_test_executable \"\${_catch_discovery_dir}/${test_executable_relative}\")" "\n" + "set(_catch_test_working_dir \"\${_catch_discovery_dir}/${test_working_dir_relative}\")" "\n" + "set(_catch_tests_file \"\${_catch_discovery_dir}/${ctest_tests_file_name}\")" "\n" + ) + set(test_executable_for_script "\${_catch_test_executable}") + set(test_working_dir_for_script "\${_catch_test_working_dir}") + set(discover_tests_script_for_script "\${_catch_discover_tests_script}") + set(ctest_tests_file_for_script "\${_catch_tests_file}") + set(test_executable_argument "\"\${_catch_test_executable}\"") + set(test_working_dir_argument "\"\${_catch_test_working_dir}\"") + set(ctest_file_argument "\"\${_catch_tests_file}\"") + set(ctest_include_file_for_property "${ctest_include_file_name}") + endif() + string(CONCAT ctest_include_content - "if(EXISTS \"$\")" "\n" - " if(NOT EXISTS \"${ctest_tests_file}\" OR" "\n" - " NOT \"${ctest_tests_file}\" IS_NEWER_THAN \"$\" OR\n" - " NOT \"${ctest_tests_file}\" IS_NEWER_THAN \"\${CMAKE_CURRENT_LIST_FILE}\")\n" - " include(\"${_CATCH_DISCOVER_TESTS_SCRIPT}\")" "\n" + "${ctest_path_setup}" + "if(EXISTS \"${test_executable_for_script}\")" "\n" + " if(NOT EXISTS \"${ctest_tests_file_for_script}\" OR" "\n" + " NOT \"${ctest_tests_file_for_script}\" IS_NEWER_THAN \"${test_executable_for_script}\" OR\n" + " NOT \"${ctest_tests_file_for_script}\" IS_NEWER_THAN \"\${CMAKE_CURRENT_LIST_FILE}\")\n" + " include(\"${discover_tests_script_for_script}\")" "\n" " catch_discover_tests_impl(" "\n" - " TEST_EXECUTABLE" " [==[" "$" "]==]" "\n" + " TEST_EXECUTABLE" " ${test_executable_argument}" "\n" " TEST_EXECUTOR" " [==[" "${crosscompiling_emulator}" "]==]" "\n" - " TEST_WORKING_DIR" " [==[" "${_WORKING_DIRECTORY}" "]==]" "\n" + " TEST_WORKING_DIR" " ${test_working_dir_argument}" "\n" " TEST_SPEC" " [==[" "${_TEST_SPEC}" "]==]" "\n" " TEST_EXTRA_ARGS" " [==[" "${_EXTRA_ARGS}" "]==]" "\n" " TEST_PROPERTIES" " [==[" "${_PROPERTIES}" "]==]" "\n" @@ -284,13 +351,13 @@ function(catch_discover_tests TARGET) " TEST_OUTPUT_DIR" " [==[" "${_OUTPUT_DIR}" "]==]" "\n" " TEST_OUTPUT_PREFIX" " [==[" "${_OUTPUT_PREFIX}" "]==]" "\n" " TEST_OUTPUT_SUFFIX" " [==[" "${_OUTPUT_SUFFIX}" "]==]" "\n" - " CTEST_FILE" " [==[" "${ctest_tests_file}" "]==]" "\n" + " CTEST_FILE" " ${ctest_file_argument}" "\n" " TEST_DL_PATHS" " [==[" "${_DL_PATHS}" "]==]" "\n" " TEST_DL_FRAMEWORK_PATHS" " [==[" "${_DL_FRAMEWORK_PATHS}" "]==]" "\n" " ADD_TAGS_AS_LABELS" " [==[" "${_ADD_TAGS_AS_LABELS}" "]==]" "\n" " )" "\n" " endif()" "\n" - " include(\"${ctest_tests_file}\")" "\n" + " include(\"${ctest_tests_file_for_script}\")" "\n" "else()" "\n" " add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)" "\n" "endif()" "\n" @@ -304,7 +371,7 @@ function(catch_discover_tests TARGET) "if(NOT CTEST_CONFIGURATION_TYPE)" "\n" " message(\"No configuration for testing specified, use '-C '.\")" "\n" "else()" "\n" - " include(\"${ctest_file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake\")" "\n" + " include(\"${ctest_config_include_file}\")" "\n" "endif()" "\n" ) file(GENERATE OUTPUT "${ctest_include_file}" CONTENT "${ctest_include_multi_content}") @@ -316,7 +383,7 @@ function(catch_discover_tests TARGET) # Add discovered tests to directory TEST_INCLUDE_FILES set_property(DIRECTORY - APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" + APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file_for_property}" ) endfunction() diff --git a/tests/TestScripts/DiscoverTests/CMakeLists.txt b/tests/TestScripts/DiscoverTests/CMakeLists.txt index 1bcefbb138..4d555a3391 100644 --- a/tests/TestScripts/DiscoverTests/CMakeLists.txt +++ b/tests/TestScripts/DiscoverTests/CMakeLists.txt @@ -13,11 +13,15 @@ if(CMAKE_VERSION GREATER_EQUAL 3.27) DL_PATHS "${CMAKE_CURRENT_LIST_DIR};${CMAKE_CURRENT_LIST_DIR}/.." ) endif() -catch_discover_tests( - tests - ADD_TAGS_AS_LABELS - DISCOVERY_MODE PRE_TEST - ${extra_args} -) +if(CATCH2_TEST_USE_RELATIVE_PATHS) + add_subdirectory(relative-registration) +else() + catch_discover_tests( + tests + ADD_TAGS_AS_LABELS + DISCOVERY_MODE PRE_TEST + ${extra_args} + ) +endif() # DISCOVERY_MODE diff --git a/tests/TestScripts/DiscoverTests/VerifyRegistration.py b/tests/TestScripts/DiscoverTests/VerifyRegistration.py index 3b7474a0ed..86558076d0 100644 --- a/tests/TestScripts/DiscoverTests/VerifyRegistration.py +++ b/tests/TestScripts/DiscoverTests/VerifyRegistration.py @@ -35,13 +35,17 @@ def get_cmake_version(): int(version_match.group(2)), int(version_match.group(3))) -def build_project(sources_dir, output_base_path, catch2_path): +def build_project(sources_dir, output_base_path, catch2_path, + use_relative_paths=False): build_dir = os.path.join(output_base_path, 'ctest-registration-test') config_cmd = ['cmake', '-B', build_dir, '-S', sources_dir, f'-DCATCH2_PATH={catch2_path}', '-DCMAKE_BUILD_TYPE=Debug'] + relative_paths_setting = 'ON' if use_relative_paths else 'OFF' + config_cmd.append( + f'-DCATCH2_TEST_USE_RELATIVE_PATHS={relative_paths_setting}') build_cmd = ['cmake', '--build', build_dir, @@ -100,23 +104,50 @@ def get_test_names(build_path: str) -> List[TestInfo]: def get_ctest_listing(build_path): old_path = os.getcwd() - os.chdir(build_path) + try: + os.chdir(build_path) + + cmd = ['ctest', '-C', 'debug', '--show-only=json-v1'] + try: + result = subprocess.run(cmd, + capture_output = True, + check = True, + text = True) + except subprocess.CalledProcessError as err: + print('Error when getting output from CTest') + print(f'cmd: {err.cmd}') + print(f'stderr: {err.stderr}') + print(f'stdout: {err.stdout}') + exit(4) + finally: + os.chdir(old_path) + + return result.stdout - cmd = ['ctest', '-C', 'debug', '--show-only=json-v1'] +def run_ctest(build_path): + cmd = ['ctest', '-C', 'debug', '--output-on-failure'] try: - result = subprocess.run(cmd, - capture_output = True, - check = True, - text = True) + subprocess.run(cmd, + capture_output = True, + check = True, + cwd = build_path, + text = True) except subprocess.CalledProcessError as err: - print('Error when getting output from CTest') + print('Error when running discovered tests') print(f'cmd: {err.cmd}') print(f'stderr: {err.stderr}') print(f'stdout: {err.stdout}') exit(4) - os.chdir(old_path) - return result.stdout +def remove_discovery_cache(build_path): + relative_registration_dir = os.path.join( + build_path, 'relative-registration') + for entry in os.scandir(relative_registration_dir): + if (entry.is_file() and + re.match(r'tests-[0-9a-f]+_(?:tests|test-list)' + r'(?:-[^.]+)?\.cmake$', + entry.name)): + os.remove(entry.path) def extract_tests_from_ctest(ctest_output) -> List[TestInfo]: ctest_response = json.loads(ctest_output) @@ -249,16 +280,9 @@ def escape_catch2_test_names(infos: List[TestInfo]): escaped.append(TestInfo(name, info.tags)) return escaped - -if __name__ == '__main__': - if len(sys.argv) != 3: - print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path') - exit(2) - catch2_path = sys.argv[1] - output_base_path = sys.argv[2] - sources_dir = os.path.dirname(os.path.abspath(sys.argv[0])) - - build_path = build_project(sources_dir, output_base_path, catch2_path) +def verify_registration(build_path, test_script_dir=None): + if test_script_dir is None: + test_script_dir = build_path raw_catch_test_names = get_test_names(build_path) catch_test_names = escape_catch2_test_names(raw_catch_test_names) @@ -280,7 +304,8 @@ def escape_catch2_test_names(infos: List[TestInfo]): exit(1) print(f"{len(catch_test_names)} tests matched in CTest listing") - test_list_names = sorted(extract_tests_list_from_ctest_script(build_path)) + test_list_names = sorted( + extract_tests_list_from_ctest_script(test_script_dir)) expected_names = sorted(info.name for info in raw_catch_test_names) if test_list_names != expected_names: print("TEST_LIST variable (tests_TESTS) does not match Catch2 test listing!") @@ -296,3 +321,34 @@ def escape_catch2_test_names(infos: List[TestInfo]): cmake_version = get_cmake_version() if cmake_version >= (3, 27): check_DL_PATHS(ctest_output) + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path') + exit(2) + catch2_path = sys.argv[1] + output_base_path = sys.argv[2] + sources_dir = os.path.dirname(os.path.abspath(sys.argv[0])) + + build_path = build_project(sources_dir, output_base_path, catch2_path) + verify_registration(build_path) + + if get_cmake_version() >= (3, 24): + build_project(sources_dir, output_base_path, catch2_path, + use_relative_paths=True) + remove_discovery_cache(build_path) + with tempfile.TemporaryDirectory( + prefix='ctest-registration-relocated-', + dir=output_base_path) as relocation_dir: + relocated_build_path = os.path.join(relocation_dir, + 'ctest-registration-test') + os.rename(build_path, relocated_build_path) + try: + verify_registration( + relocated_build_path, + os.path.join(relocated_build_path, + 'relative-registration')) + run_ctest(relocated_build_path) + finally: + os.rename(relocated_build_path, build_path) + remove_discovery_cache(build_path) diff --git a/tests/TestScripts/DiscoverTests/relative-registration/CMakeLists.txt b/tests/TestScripts/DiscoverTests/relative-registration/CMakeLists.txt new file mode 100644 index 0000000000..107c8110c7 --- /dev/null +++ b/tests/TestScripts/DiscoverTests/relative-registration/CMakeLists.txt @@ -0,0 +1,8 @@ +catch_discover_tests( + tests + ADD_TAGS_AS_LABELS + DISCOVERY_MODE PRE_TEST + USE_RELATIVE_PATHS + WORKING_DIRECTORY "$" + ${extra_args} +)