From d55f4d402197327971410c1d9469f4917c1a3960 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 29 Sep 2026 15:31:20 -0500 Subject: [PATCH 1/4] contract tests: register suites and their contract labels from the built binary The hand-maintained contract suite registry failed configure whenever a source declared a suite it did not list, so two pull requests that each pass CI, one adding a suite and one changing the registry, break configure on master once both merge, as #618 and #634 did. Its hand-written contract labels had also drifted: eight entries lacked contracts their suites deploy. After contracts_unit_test links, contracts/tools/discover_contract_suites.py reads the executable's --list_content=DOT listing and writes one CTest entry per top-level unit, so a new suite, or a test case declared outside any suite, is registered without a CMake edit. Each entry is labelled with the contracts referenced by the files that declare the unit, its test cases and their dependencies, and by the local headers those files include: accessor calls resolved through the accessor headers, and contract header includes. The sysio.bios and sysio.roa contracts that libtester deploys during default setup label every entry. If the executable cannot list itself, a single failing contract.DISCOVERY_FAILED entry carrying every label reports the error, so the build and the other tests still run. Before the executable is built, a failing contract.NOT_BUILT entry stands in. Editing the script relinks the executable, which reruns discovery. --- contracts/tests/CMakeLists.txt | 177 ++---- contracts/tools/discover_contract_suites.py | 469 ++++++++++++++++ .../tests/test_discover_contract_suites.py | 518 ++++++++++++++++++ 3 files changed, 1027 insertions(+), 137 deletions(-) create mode 100644 contracts/tools/discover_contract_suites.py create mode 100644 contracts/tools/tests/test_discover_contract_suites.py diff --git a/contracts/tests/CMakeLists.txt b/contracts/tests/CMakeLists.txt index 875399648d..9f223107a5 100644 --- a/contracts/tests/CMakeLists.txt +++ b/contracts/tests/CMakeLists.txt @@ -18,6 +18,10 @@ add_test(NAME contract_type_generator_test COMMAND ${Python3_EXECUTABLE} -m unittest discover -s ${CMAKE_SOURCE_DIR}/contracts/tools/tests -p test_generate_sysio_contract_types.py) +add_test(NAME contract_suite_discovery_test + COMMAND ${Python3_EXECUTABLE} -m unittest discover + -s ${CMAKE_SOURCE_DIR}/contracts/tools/tests + -p test_discover_contract_suites.py) # build unit test executable file(GLOB UNIT_TESTS CONFIGURE_DEPENDS "*.cpp" "*.hpp") # find all unit test suites @@ -37,140 +41,39 @@ target_link_libraries(${CONTRACT_UNIT_TEST_EXE} target_include_directories(${CONTRACT_UNIT_TEST_EXE} PRIVATE ${CMAKE_SOURCE_DIR}/libraries/opp/test/include) -# Register every top-level Boost suite as a separate CTest process. The -# processes share this executable but remain independently selectable and may -# run concurrently under ctest -j. -set(CONTRACT_SUITE_MACRO_PATTERN - "BOOST_(AUTO|FIXTURE|DATA)_TEST_SUITE[ \t\r\n]*\\([ \t\r\n]*([A-Za-z_][A-Za-z_0-9]*)[ \t\r\n]*[,)]") -function(add_contract_suite SUITE_NAME SOURCE_FILE) - cmake_parse_arguments(SUITE "" "" "ACCOUNTS" ${ARGN}) - if(SUITE_UNPARSED_ARGUMENTS) - message(FATAL_ERROR "Unexpected arguments for contract suite ${SUITE_NAME}: ${SUITE_UNPARSED_ARGUMENTS}") - endif() - if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}") - message(FATAL_ERROR "Contract suite source is missing: ${SOURCE_FILE}") - endif() - file(READ "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}" _suite_source_text) - string(REGEX MATCH - "BOOST_(AUTO|FIXTURE|DATA)_TEST_SUITE[ \t\r\n]*\\([ \t\r\n]*${SUITE_NAME}[ \t\r\n]*[,)]" - _suite_declaration - "${_suite_source_text}") - if(NOT _suite_declaration) - message(FATAL_ERROR "Contract suite ${SUITE_NAME} is not declared in ${SOURCE_FILE}") - endif() - get_property(_registered_suites GLOBAL PROPERTY CONTRACT_REGISTERED_SUITES) - if(SUITE_NAME IN_LIST _registered_suites) - message(FATAL_ERROR "Contract suite ${SUITE_NAME} is registered twice") - endif() - set_property(GLOBAL APPEND PROPERTY CONTRACT_REGISTERED_SUITES ${SUITE_NAME}) - set(_ctest_name "contract.${SUITE_NAME}") - add_test( - NAME ${_ctest_name} - COMMAND ${CONTRACT_UNIT_TEST_EXE} - --run_test=${SUITE_NAME} - --report_level=detailed - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - ) - set_tests_properties(${_ctest_name} PROPERTIES - LABELS "contract;${SUITE_ACCOUNTS}" - TIMEOUT ${CONTRACT_UNIT_TEST_TIMEOUT_SECONDS} - ) -endfunction() - -# Every runnable top-level suite is visible here with its owning source. -add_contract_suite(amm_math_tests amm_math_tests.cpp ACCOUNTS sysio.opp.common) -add_contract_suite(council_math_tests council_math_tests.cpp ACCOUNTS sysio.councl) -add_contract_suite(sysio_emissions_tests emissions_tests.cpp ACCOUNTS - sysio.dclaim sysio.epoch sysio.msig sysio.opreg - sysio.reserv sysio.system sysio.token sysio.uwrit) -add_contract_suite(t5_emissions_tests emissions_tests.cpp ACCOUNTS - sysio.dclaim sysio.epoch sysio.msig sysio.opreg - sysio.reserv sysio.system sysio.token sysio.uwrit) -add_contract_suite(sysio_producer_eligibility_tests emissions_tests.cpp ACCOUNTS - sysio.dclaim sysio.epoch sysio.msig sysio.opreg - sysio.reserv sysio.system sysio.token sysio.uwrit) -add_contract_suite(sysio_producer_score_tests emissions_tests.cpp ACCOUNTS - sysio.dclaim sysio.epoch sysio.msig sysio.opreg - sysio.reserv sysio.system sysio.token sysio.uwrit) -add_contract_suite(getpeerkeys_tests getpeerkeys_tests.cpp ACCOUNTS - sysio.msig sysio.system sysio.token) -add_contract_suite(safe_ops_tests safe_ops_tests.cpp ACCOUNTS sysio.opp.common) -add_contract_suite(sysio_authex_tests sysio.authex_tests.cpp ACCOUNTS sysio.authex) -add_contract_suite(sysio_bios_if_tests sysio.bios_inst_fin_tests.cpp ACCOUNTS sysio.bios) -add_contract_suite(sysio_chains_tests sysio.chains_tests.cpp ACCOUNTS sysio.chains) -add_contract_suite(sysio_councl_tests sysio.councl_tests.cpp ACCOUNTS - sysio.councl sysio.msig sysio.system sysio.token) -add_contract_suite(sysio_dclaim_tests sysio.dclaim_tests.cpp ACCOUNTS sysio.dclaim) -add_contract_suite(sysio_dispatch_tests sysio.dispatch_tests.cpp ACCOUNTS - sysio.authex sysio.chains sysio.chalg sysio.epoch - sysio.msgch sysio.opreg sysio.opp.common sysio.reserv - sysio.system sysio.token sysio.uwrit) -add_contract_suite(sysio_dispute_tests sysio.dispute_tests.cpp ACCOUNTS - sysio.chains sysio.chalg sysio.epoch sysio.msgch - sysio.msig sysio.opreg sysio.roa sysio.system - sysio.token sysio.uwrit) -add_contract_suite(sysio_epoch_flushwtdw_tests sysio.epoch_flushwtdw_tests.cpp ACCOUNTS - sysio.chains sysio.epoch sysio.msgch sysio.opreg - sysio.system sysio.token sysio.uwrit) -add_contract_suite(sysio_epoch_tests sysio.epoch_tests.cpp ACCOUNTS sysio.chains sysio.epoch) -add_contract_suite(sysio_system_finalizer_key_tests sysio.finalizer_key_tests.cpp ACCOUNTS - sysio.msig sysio.opreg sysio.system sysio.token) -add_contract_suite(sysio_inline_auth_tests sysio.inline_auth_tests.cpp ACCOUNTS sysio.system test.sendinline) -add_contract_suite(sysio_system_limitauth_tests sysio.limitauth_tests.cpp ACCOUNTS - sysio.msig sysio.system sysio.token) -add_contract_suite(sysio_msgch_chain_tests sysio.msgch_chain_tests.cpp ACCOUNTS - sysio.chains sysio.chalg sysio.epoch sysio.msgch - sysio.opreg sysio.roa sysio.system sysio.token - sysio.uwrit) -add_contract_suite(sysio_msgch_tests sysio.msgch_tests.cpp ACCOUNTS - sysio.chains sysio.epoch sysio.msgch) -add_contract_suite(sysio_msgch_envlog_tests sysio.msgch_tests.cpp ACCOUNTS - sysio.chains sysio.epoch sysio.msgch) -add_contract_suite(sysio_msig_tests sysio.msig_tests.cpp ACCOUNTS - sysio.msig sysio.system sysio.token test.sendinline) -add_contract_suite(sysio_opreg_tests sysio.opreg_tests.cpp ACCOUNTS - sysio.epoch sysio.opreg sysio.token) -add_contract_suite(sysio_reserve_tests sysio.reserv_tests.cpp ACCOUNTS - sysio.authex sysio.chains sysio.msgch sysio.opp.common - sysio.reserv sysio.token) -add_contract_suite(sysio_roa_tests sysio.roa_tests.cpp ACCOUNTS - sysio.authex sysio.msig sysio.roa sysio.system - sysio.token test.noop test.sendinline) -add_contract_suite(sysio_snapshot_attest_tests sysio.snapshot_attest_tests.cpp ACCOUNTS - sysio.msig sysio.system sysio.token) -add_contract_suite(sysio_system_blockinfo_tests sysio.system_blockinfo_tests.cpp ACCOUNTS - sysio.msig sysio.system sysio.token test.blockinfo_tester) -add_contract_suite(sysio_system_tests sysio.system_tests.cpp ACCOUNTS - sysio.bios sysio.msig sysio.system sysio.token) -add_contract_suite(sysio_token_tests sysio.token_tests.cpp ACCOUNTS - sysio.msig sysio.system sysio.token) -add_contract_suite(sysio_tokens_tests sysio.tokens_tests.cpp ACCOUNTS sysio.tokens) -add_contract_suite(sysio_uwrit_tests sysio.uwrit_tests.cpp ACCOUNTS sysio.uwrit) -add_contract_suite(sysio_wrap_tests sysio.wrap_tests.cpp ACCOUNTS sysio.msig sysio.wrap) -get_property(_registered_suites GLOBAL PROPERTY CONTRACT_REGISTERED_SUITES) - -# Fail at configure time when a source introduces a suite without registering it. -# Contract tests must live in a named suite: file-scope test cases cannot be -# discovered by this inventory and therefore cannot be registered separately. -set(_source_suites) -file(GLOB _contract_suite_sources CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") -foreach(_source IN LISTS _contract_suite_sources) - file(READ "${_source}" _source_text) - string(REGEX MATCHALL "${CONTRACT_SUITE_MACRO_PATTERN}" _suite_macros "${_source_text}") - foreach(_macro IN LISTS _suite_macros) - string(REGEX REPLACE "^${CONTRACT_SUITE_MACRO_PATTERN}$" "\\2" _declared_suite "${_macro}") - list(APPEND _source_suites ${_declared_suite}) - endforeach() -endforeach() -list(REMOVE_DUPLICATES _source_suites) -list(SORT _source_suites) -list(SORT _registered_suites) -set(_missing_suites ${_source_suites}) -list(REMOVE_ITEM _missing_suites ${_registered_suites}) -set(_extra_suites ${_registered_suites}) -list(REMOVE_ITEM _extra_suites ${_source_suites}) -if(_missing_suites OR _extra_suites) - message(FATAL_ERROR - "Contract suites declared but not registered: ${_missing_suites}; " - "registered but not declared: ${_extra_suites}") -endif() +# Register every top-level Boost unit as its own CTest entry, contract., so each can be selected alone and all +# can run concurrently under ctest -j while sharing this executable. The linked executable's own listing is the +# inventory, so a new suite needs no edit here. Each entry is labelled with the contracts its sources use plus the +# ones libtester deploys during default setup. +set(CONTRACT_SUITE_TEST_PREFIX contract.) +set(CONTRACT_SUITE_LABEL contract) +set(CONTRACT_SUITE_DISCOVERY_SCRIPT ${CMAKE_SOURCE_DIR}/contracts/tools/discover_contract_suites.py) +set(CONTRACT_SUITE_TESTS_FILE ${CMAKE_CURRENT_BINARY_DIR}/${CONTRACT_UNIT_TEST_EXE}_suites.cmake) +set(CONTRACT_SUITE_INCLUDE_FILE ${CMAKE_CURRENT_BINARY_DIR}/${CONTRACT_UNIT_TEST_EXE}_include.cmake) +add_custom_command(TARGET ${CONTRACT_UNIT_TEST_EXE} POST_BUILD + COMMAND ${Python3_EXECUTABLE} ${CONTRACT_SUITE_DISCOVERY_SCRIPT} + --executable $ + --working-directory ${CMAKE_BINARY_DIR} + --test-sources ${UNIT_TESTS} + --accessor-header ${CMAKE_CURRENT_SOURCE_DIR}/contracts.hpp.in + --accessor-header ${CMAKE_SOURCE_DIR}/unittests/test_contracts.hpp.in + --implicit-accessor-header ${CMAKE_SOURCE_DIR}/libraries/testing/contracts.cpp.in + --timeout ${CONTRACT_UNIT_TEST_TIMEOUT_SECONDS} + --test-argument=--report_level=detailed + --test-prefix ${CONTRACT_SUITE_TEST_PREFIX} + --base-label ${CONTRACT_SUITE_LABEL} + --output ${CONTRACT_SUITE_TESTS_FILE} + BYPRODUCTS ${CONTRACT_SUITE_TESTS_FILE} + COMMENT "Registering ${CONTRACT_UNIT_TEST_EXE} suites with CTest" + VERBATIM) +# Relink, and so rediscover, when the discovery rules change and not only when a test source does. +set_property(TARGET ${CONTRACT_UNIT_TEST_EXE} APPEND PROPERTY LINK_DEPENDS ${CONTRACT_SUITE_DISCOVERY_SCRIPT}) +# Until the executable is built, a placeholder entry fails instead of the contract suites silently not running. +file(WRITE ${CONTRACT_SUITE_INCLUDE_FILE} + "if(EXISTS \"${CONTRACT_SUITE_TESTS_FILE}\")\n" + " include(\"${CONTRACT_SUITE_TESTS_FILE}\")\n" + "else()\n" + " add_test(${CONTRACT_SUITE_TEST_PREFIX}NOT_BUILT ${CONTRACT_UNIT_TEST_EXE}_NOT_BUILT)\n" + " set_tests_properties(${CONTRACT_SUITE_TEST_PREFIX}NOT_BUILT PROPERTIES LABELS ${CONTRACT_SUITE_LABEL})\n" + "endif()\n") +set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES ${CONTRACT_SUITE_INCLUDE_FILE}) diff --git a/contracts/tools/discover_contract_suites.py b/contracts/tools/discover_contract_suites.py new file mode 100644 index 0000000000..34e6ba81f7 --- /dev/null +++ b/contracts/tools/discover_contract_suites.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +""" +Register each top-level Boost.Test unit of a test executable as its own CTest entry. + +CMake runs this after `contracts_unit_test` links. The executable's `--list_content=DOT` output is +the inventory, so adding a suite needs no CMake edit. Each entry is labelled with `contract` plus +every contract its sources deploy or compile against: accessor references such as +`contracts::liq_wasm()` and contract header includes such as ``, +scanned in the files that declare the unit, its test cases and their dependencies, and in the local +headers those files include. Contracts the test library deploys during default tester setup label +every entry. + +The labels over-approximate, since a helper in a shared header labels every suite that includes it. +That is the safe direction for selecting tests by changed contract. +""" +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +# --------------------------------------------------------------------------- +# Boost.Test `--list_content=DOT` grammar (boost/test/impl/unit_test_main.ipp, dot_content_reporter) +# --------------------------------------------------------------------------- + +DOT_NODE_RE = re.compile(r'^tu(?P\d+)\[(?P.*)\];$') +DOT_PARENT_EDGE_RE = re.compile(r'^tu(?P\d+) -> tu(?P\d+);$') +DOT_DEPENDENCY_EDGE_RE = re.compile(r'^tu(?P\d+) -> tu(?P\d+)\[[^\]]*\];$') +DOT_LABEL_RE = re.compile(r'label="(?P