Repository navigation
Test: Register Boost.Test suites discovered from the test binaries #655
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
5 commits
Select commit
Hold shift + click to select a range
d55f4d4
contract tests: register suites and their contract labels from the bu…
heifner 43aa040
cmake: register Boost.Test suites discovered from the test binaries
heifner 8ab1f52
Merge remote-tracking branch 'origin/master' into feature/contract-su…
heifner a1a0843
cmake: act on the discovery lock result and thread the config through…
heifner 0836c39
cmake: pass the configuration to the discovery test's nested configure
heifner File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,177 @@ | ||
| # Implementation of boost_test_discover_tests() (boost-test-discovery.cmake). ctest runs it from the include script | ||
| # that function generates, to list the executable's top-level units and write the CTest script registering them. | ||
|
|
||
| # ctest reads test files with no policies set; functions keep the policies in effect where they are defined. | ||
| cmake_policy(PUSH) | ||
| cmake_policy(VERSION 3.19) | ||
|
|
||
| set(_BOOST_TEST_LIST_TIMEOUT_SECONDS 60) | ||
| # Longer than one discovery can take, so a process waiting for the lock outlasts the one holding it. | ||
| set(_BOOST_TEST_LOCK_TIMEOUT_SECONDS 180) | ||
| # Stands for "no runtime" in the runtime loop: an empty element cannot be appended to an empty CMake list. | ||
| set(_BOOST_TEST_NO_RUNTIME "<none>") | ||
|
|
||
| # Quote a value as a CMake bracket argument; the trailing `]` covers a value ending in the start of the closing bracket. | ||
| function(_boost_test_quote out value) | ||
| set(equals "") | ||
| string(FIND "${value}]" "]${equals}]" found) | ||
| while(NOT found EQUAL -1) | ||
| string(APPEND equals "=") | ||
| string(FIND "${value}]" "]${equals}]" found) | ||
| endwhile() | ||
| set(${out} "[${equals}[${value}]${equals}]" PARENT_SCOPE) | ||
| endfunction() | ||
|
|
||
| # Append `<command>(<quoted arguments>)` to the script held in the caller's `script` variable. | ||
| function(_boost_test_append_command command) | ||
| set(line "${command}(") | ||
| foreach(argument IN LISTS ARGN) | ||
| _boost_test_quote(quoted "${argument}") | ||
| string(APPEND line " ${quoted}") | ||
| endforeach() | ||
| string(APPEND script "${line})\n") | ||
| set(script "${script}" PARENT_SCOPE) | ||
| endfunction() | ||
|
|
||
| # Append an entry that fails, echoing `message`, to the caller's `script`; ARGN holds its extra properties. | ||
| function(_boost_test_append_failure name cmake_command message) | ||
| _boost_test_append_command(add_test "${name}" "${cmake_command}" -E echo "${message}") | ||
| _boost_test_append_command(set_tests_properties "${name}" PROPERTIES WILL_FAIL TRUE ${ARGN}) | ||
| set(script "${script}" PARENT_SCOPE) | ||
| endfunction() | ||
|
|
||
| function(boost_test_discover_tests_impl) | ||
| # Lists arrive as single `;`-joined arguments; the ${ARGN} form splits them where PARSE_ARGV would not. | ||
| set(single_values EXECUTABLE TESTS_FILE INCLUDE_FILE FAILURE_NAME CMAKE_COMMAND WORKING_DIRECTORY NAME INCLUDE | ||
| EXCLUDE LOCK_TIMEOUT) | ||
| cmake_parse_arguments(arg "" "${single_values}" "NAME_STRIP;RUNTIMES;ARGS;PROPERTIES;COST_BY_NAME" ${ARGN}) | ||
|
|
||
| # One ctest process discovers at a time, and one that waited finds the script already current. Without the lock the | ||
| # shared files are not this process's to touch, so it keeps the current list and reports why. | ||
| if(NOT DEFINED arg_LOCK_TIMEOUT) | ||
| set(arg_LOCK_TIMEOUT ${_BOOST_TEST_LOCK_TIMEOUT_SECONDS}) | ||
| endif() | ||
| file(LOCK "${arg_TESTS_FILE}.lock" GUARD FUNCTION TIMEOUT ${arg_LOCK_TIMEOUT} RESULT_VARIABLE lock_result) | ||
| if(NOT lock_result STREQUAL "0") | ||
| set(script "") | ||
| _boost_test_append_failure("${arg_FAILURE_NAME}" "${arg_CMAKE_COMMAND}" | ||
| "cannot lock ${arg_TESTS_FILE}.lock: ${lock_result}" ${arg_PROPERTIES}) | ||
| cmake_language(EVAL CODE "${script}") | ||
| return() | ||
| endif() | ||
| # The script is stale when the executable or the include script is newer (a tie counts) or this implementation | ||
| # changed. | ||
| if(EXISTS "${arg_TESTS_FILE}" AND | ||
| NOT "${arg_EXECUTABLE}" IS_NEWER_THAN "${arg_TESTS_FILE}" AND | ||
| NOT "${arg_INCLUDE_FILE}" IS_NEWER_THAN "${arg_TESTS_FILE}" AND | ||
| "${arg_TESTS_FILE}" IS_NEWER_THAN "${CMAKE_CURRENT_FUNCTION_LIST_FILE}") | ||
| return() | ||
| endif() | ||
|
|
||
| # The report sink keeps the listing apart from anything the executable prints, and overrides | ||
| # BOOST_TEST_REPORT_SINK in the environment. | ||
| set(listing "${arg_TESTS_FILE}.listing") | ||
| file(REMOVE "${listing}") | ||
| execute_process(COMMAND "${arg_EXECUTABLE}" --list_content "--report_sink=${listing}" | ||
| WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}" TIMEOUT ${_BOOST_TEST_LIST_TIMEOUT_SECONDS} | ||
| RESULT_VARIABLE result OUTPUT_VARIABLE output ERROR_VARIABLE output) | ||
| set(text "") | ||
| if(result EQUAL 0 AND EXISTS "${listing}") | ||
| file(READ "${listing}" text) | ||
| endif() | ||
| file(REMOVE "${listing}") | ||
|
|
||
| # Boost prints one unit per line: `<name>*` when enabled, `<name> ` when disabled by default, optionally followed | ||
| # by `: <description>`, with nested units indented. Nested lines and descriptions are cut before the text becomes a | ||
| # CMake list, whose `;` and `[` handling would otherwise split or merge lines. The sink holds nothing but the | ||
| # listing, so a line without a marker is still a unit, registered enabled. | ||
| string(REGEX REPLACE "\n [^\n]*" "" text "\n${text}") | ||
| string(REGEX REPLACE "([* ]): [^\n]*" "\\1" text "${text}") | ||
| string(REGEX MATCHALL "[^\n]+" lines "${text}") | ||
|
|
||
| set(script "# Generated from `${arg_EXECUTABLE} --list_content` by boost-test-discovery-impl.cmake.\n") | ||
| set(listed 0) | ||
| set(test_names "") | ||
| set(duplicates "") | ||
| foreach(line IN LISTS lines) | ||
| if(line MATCHES "^(.+)\\*$") | ||
| set(UNIT "${CMAKE_MATCH_1}") | ||
| set(enabled TRUE) | ||
| elseif(line MATCHES "^(.+) $") | ||
| set(UNIT "${CMAKE_MATCH_1}") | ||
| set(enabled FALSE) | ||
| else() | ||
| set(UNIT "${line}") | ||
| set(enabled TRUE) | ||
| endif() | ||
| math(EXPR listed "${listed} + 1") | ||
| if((arg_INCLUDE AND NOT UNIT MATCHES "${arg_INCLUDE}") OR (arg_EXCLUDE AND UNIT MATCHES "${arg_EXCLUDE}")) | ||
| continue() | ||
| endif() | ||
|
|
||
| set(NAME "${UNIT}") | ||
| foreach(pattern IN LISTS arg_NAME_STRIP) | ||
| string(REGEX REPLACE "${pattern}" "" NAME "${NAME}") | ||
| endforeach() | ||
| set(cost "") | ||
| list(LENGTH arg_COST_BY_NAME cost_values) | ||
| set(index 0) | ||
| while(index LESS cost_values AND NOT cost) | ||
| list(GET arg_COST_BY_NAME ${index} pattern) | ||
| math(EXPR index "${index} + 1") | ||
| if(NAME MATCHES "${pattern}") | ||
| list(GET arg_COST_BY_NAME ${index} cost) | ||
| endif() | ||
| math(EXPR index "${index} + 1") | ||
| endwhile() | ||
|
|
||
| set(runtimes ${arg_RUNTIMES}) | ||
| if(NOT runtimes) | ||
| set(runtimes "${_BOOST_TEST_NO_RUNTIME}") | ||
| endif() | ||
| foreach(RUNTIME IN LISTS runtimes) | ||
| set(command "${arg_EXECUTABLE}" "--run_test=${UNIT}" ${arg_ARGS}) | ||
| if(RUNTIME STREQUAL _BOOST_TEST_NO_RUNTIME) | ||
| set(RUNTIME "") | ||
| else() | ||
| list(APPEND command "--" "--${RUNTIME}") | ||
| endif() | ||
| string(CONFIGURE "${arg_NAME}" test_name @ONLY) | ||
| if(test_name IN_LIST test_names) | ||
| list(APPEND duplicates "${test_name}") | ||
| endif() | ||
| list(APPEND test_names "${test_name}") | ||
| _boost_test_append_command(add_test "${test_name}" ${command}) | ||
| set(properties WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}" ${arg_PROPERTIES}) | ||
| if(cost) | ||
| list(APPEND properties COST ${cost}) | ||
| endif() | ||
| if(NOT enabled) | ||
| list(APPEND properties DISABLED TRUE) | ||
| endif() | ||
| _boost_test_append_command(set_tests_properties "${test_name}" PROPERTIES ${properties}) | ||
| endforeach() | ||
| endforeach() | ||
|
|
||
| if(listed EQUAL 0) | ||
| # Register the failure directly and leave no tests file, so the next ctest run discovers again. | ||
| file(REMOVE "${arg_TESTS_FILE}") | ||
| set(script "") | ||
| _boost_test_append_failure("${arg_FAILURE_NAME}" "${arg_CMAKE_COMMAND}" | ||
| "${arg_EXECUTABLE} --list_content listed no test units (exit ${result}): ${output}" | ||
| ${arg_PROPERTIES}) | ||
| cmake_language(EVAL CODE "${script}") | ||
| return() | ||
| endif() | ||
| if(duplicates) | ||
| # Units whose names coincide after NAME_STRIP would share one CTest name; report it until the executable changes. | ||
| list(REMOVE_DUPLICATES duplicates) | ||
| _boost_test_append_failure("${arg_FAILURE_NAME}" "${arg_CMAKE_COMMAND}" | ||
| "${arg_EXECUTABLE}: units map to the same test name: ${duplicates}" ${arg_PROPERTIES}) | ||
| endif() | ||
| # Written beside the target and renamed into place, so a reader never includes a partial file. | ||
| file(WRITE "${arg_TESTS_FILE}.tmp" "${script}") | ||
| file(RENAME "${arg_TESTS_FILE}.tmp" "${arg_TESTS_FILE}") | ||
| endfunction() | ||
|
|
||
| cmake_policy(POP) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| #[=======================================================================[ | ||
| boost_test_discover_tests(<target> NAME <template> [options...]) | ||
|
|
||
| Registers one CTest entry per top-level unit (a suite, or a test case declared outside any suite) of a Boost.Test | ||
| executable. The units are read from the executable's own `--list_content` output when ctest starts, the way CMake's | ||
| gtest_discover_tests(DISCOVERY_MODE PRE_TEST) works for GoogleTest, so adding or removing a suite needs no CMake edit. | ||
|
|
||
| NAME <template> CTest name. @UNIT@ is the Boost unit, @NAME@ the unit after NAME_STRIP, and | ||
| @RUNTIME@ the runtime when RUNTIMES is given. | ||
| NAME_STRIP <regex>... Patterns removed from @UNIT@, in order, to form @NAME@. | ||
| RUNTIMES <runtime>... One entry per unit per runtime, each run with `-- --<runtime>`. | ||
| INCLUDE <regex> Register only the units that match. | ||
| EXCLUDE <regex> Skip the units that match. | ||
| ARGS <arg>... Arguments after `--run_test=<unit>`. | ||
| WORKING_DIRECTORY <dir> Defaults to CMAKE_BINARY_DIR. | ||
| PROPERTIES <name> <value>... Single-valued test properties set on every entry. | ||
| COST_BY_NAME <regex> <cost>... COST for the entries whose @NAME@ matches; the first match wins. | ||
|
|
||
| A unit disabled by default is registered DISABLED. Until the executable is built a failing <target>_NOT_BUILT entry | ||
| stands in, and if the executable cannot list its units a failing <target>_DISCOVERY_FAILED entry reports why. The list | ||
| is cached until the executable or these options change. | ||
| #]=======================================================================] | ||
|
|
||
| set(_BOOST_TEST_DISCOVERY_IMPL "${CMAKE_CURRENT_LIST_DIR}/boost-test-discovery-impl.cmake") | ||
|
|
||
| function(boost_test_discover_tests TARGET) | ||
| cmake_parse_arguments(PARSE_ARGV 1 arg "" "NAME;INCLUDE;EXCLUDE;WORKING_DIRECTORY" | ||
| "NAME_STRIP;RUNTIMES;ARGS;PROPERTIES;COST_BY_NAME") | ||
| if(NOT arg_NAME OR arg_UNPARSED_ARGUMENTS) | ||
| message(FATAL_ERROR "boost_test_discover_tests(${TARGET}) needs NAME; unexpected: ${arg_UNPARSED_ARGUMENTS}") | ||
| endif() | ||
| # Lists reach the ctest-time implementation joined into single arguments, so no value may contain `;`. | ||
| foreach(option IN ITEMS NAME_STRIP RUNTIMES ARGS PROPERTIES COST_BY_NAME) | ||
| foreach(value IN LISTS arg_${option}) | ||
| if(value MATCHES ";") | ||
| message(FATAL_ERROR "boost_test_discover_tests(${TARGET}): ${option} values cannot contain `;`: ${value}") | ||
| endif() | ||
| endforeach() | ||
| endforeach() | ||
| foreach(pairs IN ITEMS PROPERTIES COST_BY_NAME) | ||
| list(LENGTH arg_${pairs} length) | ||
| math(EXPR odd "${length} % 2") | ||
| if(odd) | ||
| message(FATAL_ERROR "boost_test_discover_tests(${TARGET}): ${pairs} takes pairs, got: ${arg_${pairs}}") | ||
| endif() | ||
| endforeach() | ||
| if(NOT arg_WORKING_DIRECTORY) | ||
| set(arg_WORKING_DIRECTORY "${CMAKE_BINARY_DIR}") | ||
| endif() | ||
|
|
||
| # Each call gets its own files and placeholder names, so one executable can be discovered with different options. | ||
| get_property(call TARGET ${TARGET} PROPERTY BOOST_TEST_DISCOVERY_CALLS) | ||
| if(NOT call) | ||
| set(call 0) | ||
| endif() | ||
| math(EXPR call "${call} + 1") | ||
| set_property(TARGET ${TARGET} PROPERTY BOOST_TEST_DISCOVERY_CALLS ${call}) | ||
| set(placeholder ${TARGET}) | ||
| if(call GREATER 1) | ||
| set(placeholder ${TARGET}_${call}) | ||
| endif() | ||
| set(file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_discovery_${call}") | ||
| get_property(multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) | ||
| set(tests_file "${file_base}_tests.cmake") | ||
| if(multi_config) | ||
| set(tests_file "${file_base}_tests-$<CONFIG>.cmake") | ||
| endif() | ||
|
|
||
| set(placeholder_properties "") | ||
| foreach(value IN LISTS arg_PROPERTIES) | ||
| string(APPEND placeholder_properties " [==[${value}]==]") | ||
| endforeach() | ||
|
|
||
| # Runs when ctest reads this directory's tests; the implementation rediscovers only when the list is stale. | ||
| string(CONCAT content | ||
| "if(EXISTS [==[$<TARGET_FILE:${TARGET}>]==])\n" | ||
| " include([==[${_BOOST_TEST_DISCOVERY_IMPL}]==])\n" | ||
| " boost_test_discover_tests_impl(\n" | ||
| " EXECUTABLE [==[$<TARGET_FILE:${TARGET}>]==]\n" | ||
| " TESTS_FILE [==[${tests_file}]==]\n" | ||
| " INCLUDE_FILE \"\${CMAKE_CURRENT_LIST_FILE}\"\n" | ||
| " FAILURE_NAME [==[${placeholder}_DISCOVERY_FAILED]==]\n" | ||
| " CMAKE_COMMAND [==[${CMAKE_COMMAND}]==]\n" | ||
| " WORKING_DIRECTORY [==[${arg_WORKING_DIRECTORY}]==]\n" | ||
| " NAME [==[${arg_NAME}]==]\n" | ||
| " NAME_STRIP [==[${arg_NAME_STRIP}]==]\n" | ||
| " RUNTIMES [==[${arg_RUNTIMES}]==]\n" | ||
| " INCLUDE [==[${arg_INCLUDE}]==]\n" | ||
| " EXCLUDE [==[${arg_EXCLUDE}]==]\n" | ||
| " ARGS [==[${arg_ARGS}]==]\n" | ||
| " PROPERTIES [==[${arg_PROPERTIES}]==]\n" | ||
| " COST_BY_NAME [==[${arg_COST_BY_NAME}]==])\n" | ||
| " if(EXISTS [==[${tests_file}]==])\n" | ||
| " include([==[${tests_file}]==])\n" | ||
| " endif()\n" | ||
| "else()\n" | ||
| " add_test([==[${placeholder}_NOT_BUILT]==] [==[${placeholder}_NOT_BUILT]==])\n") | ||
| if(arg_PROPERTIES) | ||
| string(APPEND content | ||
| " set_tests_properties([==[${placeholder}_NOT_BUILT]==] PROPERTIES${placeholder_properties})\n") | ||
| endif() | ||
| string(APPEND content "endif()\n") | ||
|
|
||
| # A multi-config generator gets one script per configuration, chosen by the configuration ctest runs (-C). | ||
| if(multi_config) | ||
| foreach(config IN LISTS CMAKE_CONFIGURATION_TYPES) | ||
| file(GENERATE OUTPUT "${file_base}_include-${config}.cmake" CONTENT "${content}" | ||
| CONDITION $<CONFIG:${config}>) | ||
| endforeach() | ||
| file(WRITE "${file_base}_include.cmake" "include(\"${file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake\")\n") | ||
| else() | ||
| file(GENERATE OUTPUT "${file_base}_include.cmake" CONTENT "${content}") | ||
| endif() | ||
| set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES "${file_base}_include.cmake") | ||
| endfunction() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
cmake_parse_arguments(... ${ARGN})reparses every function argument through CMake list syntax, whose elements cannot contain unmatched[or]. A valid checkout/build path such as/tmp/pr655]reviewtherefore corrupts the later argument boundaries; on this headarg_TESTS_FILEbecomes empty,file(REMOVE)warns about an empty filename, and discovery registers no tests. Please preserve scalar arguments with positional/PARSE_ARGVhandling and explicitly expand only the serialized multi-value options._boost_test_append_commandalso iterates itsARGNas a list at line 28, so path-valued emitted arguments need the same lossless handling.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks, that's accurate, but I'd rather not handle it here. A checkout or build path with an unmatched
[or]breaks CMake list handling generally, not just in this module: thefile(GLOB UNIT_TESTS ...)→add_executable(... ${UNIT_TESTS})pattern inunittests/andcontracts/tests/merges every source path into one element and configure fails with "Cannot find source file" in such a directory, so discovery never runs. CMake's ownGoogleTestAddTests.cmakeparses with the same${ARGN}form. Making this lossless would mean one argument per list element,PARSE_ARGV, andARGV-indexed command building, all for paths the build can't use, so I'm leaving it as is.