From 520b4edb5b3729d39d3da6ba4643b04d9480e016 Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 23 Oct 2026 13:56:03 +0800 Subject: [PATCH 1/6] fix(random): avoid signed range overflow --- docs/reference/utility.md | 8 +++ docs/reference/utility_CN.md | 8 +++ src/core/random/random_generator.hpp | 8 +-- tests/core/CMakeLists.txt | 2 +- tests/core/README.md | 2 + tests/core/README_CN.md | 2 + tests/core/random_generator.cpp | 102 +++++++++++++++++++++++++++ 7 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 tests/core/random_generator.cpp diff --git a/docs/reference/utility.md b/docs/reference/utility.md index ce3692b42..cb05a2db3 100644 --- a/docs/reference/utility.md +++ b/docs/reference/utility.md @@ -32,6 +32,14 @@ Public headers now have their single source in `src/plugins/utility/`. CMake cop **Breaking source change:** `ark/utility/{guid,timer,schedule,compress}_module_impl.hpp` and `ark/utility/utility_plugin.hpp` are removed from the SDK. Implementations are defined in their cpp files; the plugin type and interface-returning factories are private. Replace direct implementation construction with public interface lookup or an application-owned implementation. Handwritten `-I /include` consumers must use target-provided include roots. Rebuild the plugin and consumers together; unchanged public virtual functions and exported entry signatures do not establish compatibility with old implementation-class binaries. +## Core random helper + +`` provides `ark::random_generator` through `ark::core`; it does not require loading the Utility plugin. The signed `random(int, int)` overload accepts any pair of `int` endpoints, including `INT_MIN` and `INT_MAX`, without overflowing the range calculation. Different endpoints produce a value in `[lower, upper)` after ordering the endpoints. Equal endpoints return that value without advancing the generator. + +Range selection retains the existing modulo mapping, seed sequence and draw count. Each call with different endpoints consumes one draw; reversing the endpoints produces the same sequence from the same seed. The unsigned overloads retain their existing behavior. + ## Verification `check-typed-examples` loads the actual Utility DSO with `timers`, `module_lifecycle` and `entity_operations`. It verifies all four interface RTTI keys and manager ownership, GUID generation/parsing, zlib/Zstandard binary round trips, a timer triggered by owner-thread time advancement, and normal/idempotent shutdown. `check-app-presets` exercises the shipped launcher preset; `check-core` includes both when applications, examples, Kernel and Utility are enabled. The independent `tests/consumer` project's `check-sdk` compiles public/helper paths with runtime plugins disabled and rejects visible implementation or assembly headers. These are correctness checks, not performance or arbitrary callback-lifetime acceptance. + +`check-core` also includes standalone random-header compilation and deterministic random-range regressions for extreme, mixed-sign, reversed and equal endpoints. See [core regressions](../../tests/core/README.md) for the build commands. diff --git a/docs/reference/utility_CN.md b/docs/reference/utility_CN.md index 31942906b..5445ed351 100644 --- a/docs/reference/utility_CN.md +++ b/docs/reference/utility_CN.md @@ -32,6 +32,14 @@ const auto entity = guids->create_guid(); **源码不兼容变更：** SDK 移除 `ark/utility/{guid,timer,schedule,compress}_module_impl.hpp` 和 `ark/utility/utility_plugin.hpp`。实现类定义在各自 cpp 文件中，插件类型及返回接口的工厂属于私有装配。将直接构造实现类的代码改为公开接口查找，或应用自行维护实现。仅手写 `-I /include` 的调用需改用目标传递的包含目录。插件和使用方应一起重新编译；公开虚函数及动态入口签名保持不变，不代表兼容旧实现类二进制。 +## Core 随机数工具 + +`` 通过 `ark::core` 提供 `ark::random_generator`，无需加载 Utility 插件。有符号 `random(int, int)` 重载接受任意两个 `int` 端点，包括 `INT_MIN` 和 `INT_MAX`，范围计算不会溢出。端点不同时，先按大小排序，再返回 `[lower, upper)` 内的值。端点相同时，直接返回该值，不推进生成器状态。 + +范围选择保留现有的取模映射、种子序列和生成次数。端点不同的每次调用消耗一个随机数；使用相同种子并交换端点，得到相同序列。无符号重载的行为保持不变。 + ## 验证 `check-typed-examples` 与 timers、module_lifecycle、entity_operations 一起加载真实 Utility DSO，验证四个接口的 RTTI 注册键与管理器归属、GUID 生成与解析、zlib/Zstandard 二进制往返、由所属线程时间推进触发的定时器，以及正常和重复停机。`check-app-presets` 使用仓库启动器预设；应用、示例、Kernel、Utility 开启时，`check-core` 包含这两组检查。独立 `tests/consumer` 项目的 `check-sdk` 在关闭运行时插件时编译公开接口与辅助头路径，并拒绝可见的实现或装配头。这些检查验证正确性，不代表性能或任意回调生命周期验收。 + +`check-core` 还包含随机数头文件的独立编译，以及极值、异号、逆序和相等端点的确定性随机范围回归。构建命令见[核心回归](../../tests/core/README_CN.md)。 diff --git a/src/core/random/random_generator.hpp b/src/core/random/random_generator.hpp index eb77d44ed..fe8ffa4ad 100644 --- a/src/core/random/random_generator.hpp +++ b/src/core/random/random_generator.hpp @@ -128,10 +128,10 @@ class random_generator // max_range = iMinSwap; } - int range = max_range - min_range; - int rand_result = random((uint32_t)range); + int64_t range = static_cast(max_range) - min_range; + uint32_t rand_result = random(static_cast(range)); - return rand_result + min_range; + return static_cast(static_cast(min_range) + rand_result); } int get_random_count() { @@ -143,4 +143,4 @@ class random_generator int get_count_; }; -} // namespace ark \ No newline at end of file +} // namespace ark diff --git a/tests/core/CMakeLists.txt b/tests/core/CMakeLists.txt index 3999aa898..77957c22e 100644 --- a/tests/core/CMakeLists.txt +++ b/tests/core/CMakeLists.txt @@ -6,7 +6,7 @@ target_link_libraries(ark_core_header_checks PRIVATE ark::core) add_executable(ark_core_tests main.cpp strings.cpp containers.cpp templates.cpp callbacks.cpp header_regressions.cpp cxx20_language.cpp concurrency.cpp plugin_manager_startup.cpp data_strings.cpp node_data_storage.cpp - object_vectors.cpp plugin_construction.cpp module_factory.cpp) + object_vectors.cpp plugin_construction.cpp module_factory.cpp random_generator.cpp) target_link_libraries(ark_core_tests PRIVATE ark::core ark::kernel ark::utility) add_dependencies(ark_core_tests ark_core_header_checks) add_library(ark_construction_header_checks OBJECT headers/module_base.cpp headers/plugin_base.cpp) diff --git a/tests/core/README.md b/tests/core/README.md index a6a739a94..227bec957 100644 --- a/tests/core/README.md +++ b/tests/core/README.md @@ -27,6 +27,8 @@ Initialize Protobuf as described for application builds. The C++20 language regressions check the exact six built-in integer classifications, qualifier and reference handling, constant values, exception contracts and fixed UID allocation outputs. The standalone array-map header check also rejects reintroduced legacy language macros. +The random-generator regressions include the public header first and twice. They preserve fixed raw/signed/unsigned sequences, final seeds and draw counts, and check half-open signed ranges at `INT_MIN`/`INT_MAX`, mixed-sign and adjacent endpoints, reversed endpoints and equal endpoints. Equal endpoints return their value without consuming a draw. These cases are part of `ark_core_tests` and `check-core`; a build with `-fsanitize=undefined -fno-sanitize-recover=all` also checks the range arithmetic with UndefinedBehaviorSanitizer. See [random-helper behavior](../../docs/reference/utility.md#core-random-helper). + With `ARK_BUILD_CONFIG=ON` (the default for core tests), `check-application-manifest` checks TOML/XML equivalence, defaults, strict fields, invalid input, diagnostics and owning results. It is also part of `check-core`. diff --git a/tests/core/README_CN.md b/tests/core/README_CN.md index a6d7c83b6..c1aa8868c 100644 --- a/tests/core/README_CN.md +++ b/tests/core/README_CN.md @@ -17,6 +17,8 @@ cmake --build core-build --target check-core C++20 语言回归验证六种精确整数类型分类、限定符与引用处理、常量值、异常契约和固定 UID 位分配结果。独立 array-map 头文件检查还会拒绝重新引入旧语言兼容宏。 +随机数回归将公开头文件放在首个包含位置，并重复包含两次。测试保留固定的原始、有符号及无符号随机序列、最终种子和生成次数，并验证 `INT_MIN`／`INT_MAX` 极值、异号、相邻、逆序及相等端点的有符号半开区间。端点相同时直接返回该值，不消耗随机数。这些用例属于 `ark_core_tests` 和 `check-core`；使用 `-fsanitize=undefined -fno-sanitize-recover=all` 构建时，还会通过 UndefinedBehaviorSanitizer 检查范围运算。参见[随机数工具行为](../../docs/reference/utility_CN.md#core-随机数工具)。 + 公共头检查将 `` 放在首个包含位置，并重复包含两次；编译期断言 验证其无符号 32 位底层类型及所有已声明的错误值。 diff --git a/tests/core/random_generator.cpp b/tests/core/random_generator.cpp new file mode 100644 index 000000000..65627efee --- /dev/null +++ b/tests/core/random_generator.cpp @@ -0,0 +1,102 @@ +// Copyright (c) 2026 OpenArkStudio. Licensed under Apache-2.0. See LICENSE. +#include + +// Keep the first inclusion separate to check standalone inclusion and the include guard. +#include +#include +#include + +#include "../network/doctest.h" + +#include + +TEST_CASE("Random generator retains its seed sequence and resets its count") { + ark::random_generator generator; + CHECK(generator.get_seed() == 0); + CHECK(generator.get_random_count() == 0); + generator.set_seed(1); + + const std::array expected{476707713, 1186278907, 505671508, 2137716191, 936145377, 1215825599}; + for (const auto value : expected) + { + CHECK(generator.random() == value); + } + CHECK(generator.get_seed() == 3535774167u); + CHECK(generator.get_random_count() == 6); + generator.set_seed(1); + CHECK(generator.get_seed() == 1); + CHECK(generator.get_random_count() == 0); + CHECK(generator.random() == expected.front()); +} + +TEST_CASE("Signed random ranges retain legacy in-range values and unsigned overloads") { + ark::random_generator signed_values; + ark::random_generator unsigned_values; + signed_values.set_seed(1); + unsigned_values.set_seed(1); + + const std::array expected_signed{13, 7, 8, 91, 77, 99}; + const std::array expected_unsigned{23, 17, 18, 101, 87, 109}; + for (std::size_t index = 0; index < expected_signed.size(); ++index) + { + CHECK(signed_values.random(-100, 100) == expected_signed[index]); + CHECK(unsigned_values.random(10u, 110u) == expected_unsigned[index]); + } + CHECK(signed_values.get_seed() == unsigned_values.get_seed()); + CHECK(signed_values.get_random_count() == unsigned_values.get_random_count()); + CHECK(unsigned_values.random(0u) == 0); + CHECK(unsigned_values.random(10u, 10u) == 10); + CHECK(unsigned_values.get_seed() == 3535774167u); + CHECK(unsigned_values.get_random_count() == 6); +} + +TEST_CASE("Signed random ranges decode full-width spans without overflow") { + ark::random_generator generator; + generator.set_seed(1); + const std::array expected{-1670775935, -961204741, -1641812140, -9767457, -1211338271, -931658049}; + for (const auto value : expected) + { + CHECK(generator.random(std::numeric_limits::min(), std::numeric_limits::max()) == value); + } + CHECK(generator.get_seed() == 3535774167u); + CHECK(generator.get_random_count() == 6); +} + +TEST_CASE("Signed random ranges preserve half-open reversed and equal bounds") { + constexpr int lowest = std::numeric_limits::min(); + constexpr int highest = std::numeric_limits::max(); + const std::array, 12> ranges{ + {{lowest, highest}, {lowest, 0}, {-1, highest}, {lowest, 1}, {-100, 100}, {-100, -1}, {0, highest}, + {lowest, lowest + 1}, {highest - 1, highest}, {lowest, lowest}, {highest, highest}, {0, 0}}}; + for (const auto& bounds : ranges) + { + CAPTURE(bounds[0]); + CAPTURE(bounds[1]); + for (const std::uint32_t seed : {0u, 1u, 123456789u, std::numeric_limits::max()}) + { + ark::random_generator forward; + ark::random_generator reversed; + forward.set_seed(seed); + reversed.set_seed(seed); + for (int index = 0; index < 64; ++index) + { + const int value = forward.random(bounds[0], bounds[1]); + CHECK(value == reversed.random(bounds[1], bounds[0])); + if (bounds[0] == bounds[1]) + { + CHECK(value == bounds[0]); + CHECK(forward.get_seed() == seed); + CHECK(forward.get_random_count() == 0); + } + else + { + CHECK(value >= bounds[0]); + CHECK(value < bounds[1]); + CHECK(forward.get_random_count() == index + 1); + } + CHECK(forward.get_seed() == reversed.get_seed()); + CHECK(forward.get_random_count() == reversed.get_random_count()); + } + } + } +} From 7cec356d8d16ed7635c47ded4e7e07e40cfd2a79 Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 23 Oct 2026 13:57:01 +0800 Subject: [PATCH 2/6] fix(compress): bound per-call codec output buffers --- .github/workflows/ci.yml | 5 +- CMakeLists.txt | 8 + docs/dependencies.md | 1 + docs/dependencies_CN.md | 1 + docs/reference/utility.md | 26 +++ docs/reference/utility_CN.md | 20 ++ src/core/compress/lz4.hpp | 44 ++-- src/core/compress/zlib.hpp | 34 ++- src/core/compress/zstd.hpp | 90 +++++--- tests/ci/package-dependencies.test.mjs | 22 +- tests/compression/CMakeLists.txt | 19 ++ tests/compression/compression.cpp | 292 +++++++++++++++++++++++++ tests/compression/headers/lz4.cpp | 2 + tests/compression/headers/zlib.cpp | 2 + tests/compression/headers/zstd.cpp | 2 + tests/core/README.md | 17 ++ tests/core/README_CN.md | 14 ++ vcpkg.json | 8 + 18 files changed, 547 insertions(+), 60 deletions(-) create mode 100644 tests/compression/CMakeLists.txt create mode 100644 tests/compression/compression.cpp create mode 100644 tests/compression/headers/lz4.cpp create mode 100644 tests/compression/headers/zlib.cpp create mode 100644 tests/compression/headers/zstd.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5197e1c9e..c7ca2a553 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,7 +175,7 @@ jobs: id: vcpkg-cache env: VCPKG_CACHE_REVISION: 0699a19d0c6386247ce50d4dedbb8217d484d536 - VCPKG_CACHE_FEATURES: 'application;boost;etcd-client' + VCPKG_CACHE_FEATURES: 'application;boost;compression-tests;etcd-client' VCPKG_CACHE_DEFAULT_FEATURES: 'true' VCPKG_CACHE_PROFILE: applications run: node .github/scripts/vcpkg-cache.mjs @@ -198,7 +198,7 @@ jobs: run: >- cmake -S . -B ci-build -G Ninja "-DCMAKE_TOOLCHAIN_FILE=$env:VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" - "-DVCPKG_MANIFEST_FEATURES=application;boost;etcd-client" + "-DVCPKG_MANIFEST_FEATURES=application;boost;compression-tests;etcd-client" -DCMAKE_BUILD_TYPE=Debug ${{ runner.os == 'Windows' && '-DVCPKG_APPLOCAL_DEPS=OFF' || '' }} -DARK_BUILD_APPLICATIONS=ON @@ -207,6 +207,7 @@ jobs: -DARK_BUILD_PROTO=ON -DARK_BUILD_TESTS=ON -DARK_BUILD_CORE_TESTS=ON + -DARK_BUILD_COMPRESSION_TESTS=ON -DARK_BUILD_EXAMPLES=ON -DASIO_STANDALONE=OFF - name: Save vcpkg package archives diff --git a/CMakeLists.txt b/CMakeLists.txt index 4cfc24c11..11cb0f4dd 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ endif() option(ARK_BUILD_TESTS "Enable build tests" ON) option(ARK_BUILD_CORE_TESTS "Build core C++20 regressions" ${ARK_BUILD_TESTS}) +option(ARK_BUILD_COMPRESSION_TESTS "Build bounded compression regressions (zlib, Zstandard and LZ4)" OFF) option(ARK_BUILD_EXAMPLES "Enable build examples" ON) set(ark_config_default OFF) if(ARK_BUILD_APPLICATIONS OR ARK_BUILD_CORE_TESTS) @@ -301,6 +302,13 @@ if(ARK_BUILD_CORE_TESTS) endif() endif() +if(ARK_BUILD_COMPRESSION_TESTS) + add_subdirectory(tests/compression) + if(TARGET check-core) + add_dependencies(check-core check-compression) + endif() +endif() + if(ARK_BUILD_BENCHMARKS OR ARK_BUILD_DATABASE_BENCHMARKS) add_subdirectory(benchmarks) endif() diff --git a/docs/dependencies.md b/docs/dependencies.md index f62db2791..fe9ccc2a9 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -12,6 +12,7 @@ source tree or dependency submodule initialization. | Boost networking | `boost` (default) | `boost-asio`, `boost-beast` | | Manifest reader | `config` (default) | `tomlplusplus` → `tomlplusplus::tomlplusplus` | | Applications | `application` | `args` → `taywee::args`, Protobuf, zlib, Zstandard | +| Compression regressions | `compression-tests` | zlib, Zstandard, LZ4; `ARK_BUILD_COMPRESSION_TESTS=ON` | | etcd client plugin | `etcd-client` | `nlohmann-json` → `nlohmann_json::nlohmann_json` | | Native database development | `database` | OpenSSL | diff --git a/docs/dependencies_CN.md b/docs/dependencies_CN.md index d466cfb8f..33143bfcd 100644 --- a/docs/dependencies_CN.md +++ b/docs/dependencies_CN.md @@ -11,6 +11,7 @@ | Boost 网络 | `boost`（默认） | `boost-asio`、`boost-beast` | | 应用清单读取 | `config`（默认） | `tomlplusplus` → `tomlplusplus::tomlplusplus` | | 应用 | `application` | `args` → `taywee::args`、Protobuf、zlib、Zstandard | +| 压缩回归 | `compression-tests` | zlib、Zstandard、LZ4；`ARK_BUILD_COMPRESSION_TESTS=ON` | | etcd 客户端插件 | `etcd-client` | `nlohmann-json` → `nlohmann_json::nlohmann_json` | | 原生数据库开发 | `database` | OpenSSL | diff --git a/docs/reference/utility.md b/docs/reference/utility.md index ce3692b42..25e96a250 100644 --- a/docs/reference/utility.md +++ b/docs/reference/utility.md @@ -26,6 +26,32 @@ Pointers are borrowed from the loaded plugin. Keep the owning plugin alive while The plugin keeps installation order GUID, timer, schedule, compression and reverses that order during uninstallation. Timer `pre_shut()` still shuts down its time wheel. GUID layout, compression levels/algorithms and scheduler behavior are unchanged. The library stem remains `ark_utility_plugin`, with the existing platform extension and Debug `_d` postfix. +## Compression buffers and concurrency + +The core helpers ``, `` and +`` use an owned heap buffer for each call. Successful calls +append to the supplied output string; rejected input leaves it unchanged. Callers +must supply a readable input range and synchronize any shared output string. +The existing compression policy requires the codec's worst-case output bound to +be strictly below 1 MiB. Decompression accepts at most 1 MiB of output, including +exactly 1 MiB. Zstandard requires a known frame content size and rejects larger, +unknown-size or invalid frames before decoding. Invalid compressed data returns +`false`. LZ4's existing empty-block decompression result remains `false`. + +zlib, LZ4 and Zstandard normal calls have no shared writable output buffer. +Zstandard dictionary calls still mutate their instance's cached contexts: use a +separate `zstd_compress` instance per thread, or externally serialize dictionary +initialization, dictionary calls and destruction. Dictionary initialization, +ownership and copy behavior retain their existing contract. This change does not +establish same-instance dictionary concurrency or plugin unload safety. + +Enable `ARK_BUILD_COMPRESSION_TESTS=ON` and the vcpkg `compression-tests` feature +to run `cmake --build --target check-compression`. The target is also +part of `check-core` when enabled; Applications CI enables it explicitly. It +checks the mapped public headers, capacity boundaries, malformed frames, append +behavior and parallel round trips with separate dictionary instances. LZ4 remains +a direct core helper; the Utility module still selects zlib or Zstandard. + ## Source and build-tree SDK migration Public headers now have their single source in `src/plugins/utility/`. CMake copies seven explicitly listed public headers into the build tree's `src/plugins/sdk-include/ark/utility/`: the four module interfaces plus `scheduler.hpp`, `ctpl.h` and `ctpl_stl.h`. Consume `ark::utility` to retain all existing `` public/helper include paths, including with runtime plugins disabled. Do not add the private plugin directory to consumer include paths. The upstream thread-pool headers retain their original bytes and formatting rules. For installed consumption, use the [CMake package](../get-started/installation.md#install-and-consume-the-cmake-package) with `find_package(ark CONFIG REQUIRED COMPONENTS utility)` and the same `ark::utility` target. Installed public headers retain `` paths; the runtime plugin still needs to be selected in the application manifest. diff --git a/docs/reference/utility_CN.md b/docs/reference/utility_CN.md index 31942906b..62f862026 100644 --- a/docs/reference/utility_CN.md +++ b/docs/reference/utility_CN.md @@ -26,6 +26,26 @@ const auto entity = guids->create_guid(); 插件按 GUID、timer、schedule、compress 顺序安装模块，卸载顺序相反。timer 的 `pre_shut()` 仍关闭时间轮。GUID 布局、压缩级别与算法、scheduler 行为不变。动态库主体名称仍为 `ark_utility_plugin`，保留平台扩展名和 Debug `_d` 后缀。 +## 压缩缓冲区与并发 + +核心辅助头 ``、`` 和 +`` 为每次调用分配独立的堆缓冲区。成功时向输出字符串追加结果； +拒绝输入时保留输出字符串。调用方须提供可读的输入范围，并同步对共享输出字符串的访问。 +压缩保留既有限制：算法计算的最坏输出大小必须严格小于 1 MiB。解压输出最大为 +1 MiB，包含恰好 1 MiB 的情况。Zstandard 要求帧声明已知内容大小；大小超限、未知或 +无效的帧在解码前被拒绝。无效压缩数据返回 `false`。LZ4 对空块解压仍返回 `false`。 + +zlib、LZ4 和 Zstandard 普通调用不再共享可写输出缓冲区。Zstandard 字典调用仍会修改 +实例缓存的上下文；每个线程须使用独立的 `zstd_compress` 实例，或由调用方串行执行 +字典初始化、字典调用和析构。字典初始化、所有权和复制行为保留既有约定。 +本次修改不建立同一实例的字典并发保证，也不建立插件卸载安全保证。 + +启用 `ARK_BUILD_COMPRESSION_TESTS=ON` 和 vcpkg 的 `compression-tests` feature 后， +执行 `cmake --build --target check-compression`。启用时，`check-core` 也包含 +此目标；Applications CI 会显式开启。检查覆盖映射后的公开头、容量边界、损坏帧、追加 +语义，以及使用独立字典实例的并行往返。LZ4 仍通过核心辅助头直接使用；Utility 模块 +继续只选择 zlib 或 Zstandard。 + ## 源码与构建树 SDK 迁移 公开头文件的唯一源码现在位于 `src/plugins/utility/`。CMake 将显式列出的七个公开头复制到构建树的 `src/plugins/sdk-include/ark/utility/`：四个模块接口，以及 `scheduler.hpp`、`ctpl.h`、`ctpl_stl.h`。通过 `ark::utility` 消费，保留现有 `` 公开接口与辅助头路径，关闭运行时插件时同样有效。不要把私有插件目录加入使用方的包含路径。上游线程池头文件保留原字节和格式规则。安装后的消费方式见 [CMake 包安装说明](../get-started/installation_CN.md#安装并消费-cmake-包)：使用 `find_package(ark CONFIG REQUIRED COMPONENTS utility)` 和相同的 `ark::utility` 目标。安装头保留 `` 路径；运行时插件仍需在应用清单中选择。 diff --git a/src/core/compress/lz4.hpp b/src/core/compress/lz4.hpp index 7333b18e7..bf2688b80 100644 --- a/src/core/compress/lz4.hpp +++ b/src/core/compress/lz4.hpp @@ -20,6 +20,10 @@ #pragma once +#include +#include +#include + #include #include "ark/macros.hpp" @@ -33,32 +37,46 @@ class lz4_compress public: static bool compress( IN const char* in_data, IN std::size_t in_size, IN int c_level, OUT std::string& compress_buff) { - static char pre_buffer[PRE_ALLOCATED_LZ4_BUFF_SIZE] = {0}; - memset(pre_buffer, 0, sizeof(pre_buffer)); + if (in_size > static_cast((std::numeric_limits::max)())) + { + return false; + } - const int pre_buff_size = LZ4_compressBound(in_size); - if (pre_buff_size >= PRE_ALLOCATED_LZ4_BUFF_SIZE) + const int source_size = static_cast(in_size); + const int pre_buff_size = LZ4_compressBound(source_size); + if (pre_buff_size <= 0 || pre_buff_size >= PRE_ALLOCATED_LZ4_BUFF_SIZE) { return false; } - const int compressed_size = LZ4_compress_fast(in_data, pre_buffer, in_size, pre_buff_size, c_level); - ARK_ASSERT_RET_VAL(compressed_size > 0, false); + std::vector pre_buffer(pre_buff_size); + const int compressed_size = LZ4_compress_fast(in_data, pre_buffer.data(), source_size, pre_buff_size, c_level); + if (compressed_size <= 0) + { + return false; + } - compress_buff.append(pre_buffer, compressed_size); + compress_buff.append(pre_buffer.data(), compressed_size); return true; } static bool decompress(IN const char* in_data, IN std::size_t in_size, OUT std::string& decompress_buff) { - static char pre_buffer[PRE_ALLOCATED_LZ4_BUFF_SIZE] = {0}; - memset(pre_buffer, 0, sizeof(pre_buffer)); + if (in_size > static_cast((std::numeric_limits::max)())) + { + return false; + } - int const decompressed_size = LZ4_decompress_safe(in_data, pre_buffer, in_size, PRE_ALLOCATED_LZ4_BUFF_SIZE); - ARK_ASSERT_RET_VAL(decompressed_size > 0, false); + std::vector pre_buffer(PRE_ALLOCATED_LZ4_BUFF_SIZE); + int const decompressed_size = LZ4_decompress_safe( + in_data, pre_buffer.data(), static_cast(in_size), static_cast(pre_buffer.size())); + if (decompressed_size <= 0) + { + return false; + } - decompress_buff.append(pre_buffer, decompressed_size); + decompress_buff.append(pre_buffer.data(), decompressed_size); return true; } }; -} // namespace ark \ No newline at end of file +} // namespace ark diff --git a/src/core/compress/zlib.hpp b/src/core/compress/zlib.hpp index 103b2aa54..570d2ec4f 100644 --- a/src/core/compress/zlib.hpp +++ b/src/core/compress/zlib.hpp @@ -20,6 +20,10 @@ #pragma once +#include +#include +#include + #include #include "ark/macros.hpp" @@ -37,39 +41,47 @@ class zlib_compress final public: static bool compress( IN const char* in_data, IN std::size_t in_size, IN uint8_t compress_level, OUT std::string& compress_buff) { - static unsigned char pre_buffer[PRE_ALLOCATED_ZLIB_BUFF_SIZE] = {0}; - memset(pre_buffer, 0, sizeof(pre_buffer)); + if (in_size >= PRE_ALLOCATED_ZLIB_BUFF_SIZE || in_size > (std::numeric_limits::max)()) + { + return false; + } - unsigned long pre_buff_size = compressBound(in_size); // calc needed size + uLongf pre_buff_size = compressBound(static_cast(in_size)); // calc needed size if (pre_buff_size >= PRE_ALLOCATED_ZLIB_BUFF_SIZE) { return false; } - int ret = compress2(pre_buffer, &pre_buff_size, (unsigned char*)in_data, in_size, compress_level); + std::vector pre_buffer(pre_buff_size); + int ret = compress2(pre_buffer.data(), &pre_buff_size, reinterpret_cast(in_data), + static_cast(in_size), compress_level); if (ret != Z_OK) { return false; } - compress_buff.append((char*)pre_buffer, pre_buff_size); + compress_buff.append(reinterpret_cast(pre_buffer.data()), pre_buff_size); return true; } static bool decompress(IN const char* in_data, IN std::size_t in_size, OUT std::string& decompress_buff) { - static unsigned char pre_buffer[PRE_ALLOCATED_ZLIB_BUFF_SIZE] = {0}; - memset(pre_buffer, 0, sizeof(pre_buffer)); + if (in_size > (std::numeric_limits::max)()) + { + return false; + } - unsigned long pre_buff_size = PRE_ALLOCATED_ZLIB_BUFF_SIZE; - int ret = uncompress2(pre_buffer, &pre_buff_size, (unsigned char*)in_data, (uLong*)&in_size); + std::vector pre_buffer(PRE_ALLOCATED_ZLIB_BUFF_SIZE); + uLongf pre_buff_size = static_cast(pre_buffer.size()); + uLong source_size = static_cast(in_size); + int ret = uncompress2(pre_buffer.data(), &pre_buff_size, reinterpret_cast(in_data), &source_size); if (ret != Z_OK) { return false; } - decompress_buff.append((char*)pre_buffer, pre_buff_size); + decompress_buff.append(reinterpret_cast(pre_buffer.data()), pre_buff_size); return true; } }; -} // namespace ark \ No newline at end of file +} // namespace ark diff --git a/src/core/compress/zstd.hpp b/src/core/compress/zstd.hpp index 5b5131488..e3fff5c04 100644 --- a/src/core/compress/zstd.hpp +++ b/src/core/compress/zstd.hpp @@ -20,8 +20,11 @@ #pragma once +#include +#include #include #include +#include #include @@ -71,8 +74,10 @@ class zstd_compress final /// simple compress and decompress bool normal_compress( IN const char* in_data, IN std::size_t in_size, IN uint8_t compress_level, OUT std::string& compressed_buff) { - static char pre_buffer[PRE_ALLOCATED_ZSTD_BUFF_SIZE] = {0}; - memset(pre_buffer, 0, sizeof(pre_buffer)); + if (in_size >= PRE_ALLOCATED_ZSTD_BUFF_SIZE) + { + return false; + } size_t const pre_buff_size = ZSTD_compressBound(in_size); // calc needed size if (pre_buff_size >= PRE_ALLOCATED_ZSTD_BUFF_SIZE) @@ -80,32 +85,41 @@ class zstd_compress final return false; } - std::size_t const compressed_size = ZSTD_compress(pre_buffer, pre_buff_size, in_data, in_size, compress_level); - ARK_ASSERT_RET_VAL(!ZSTD_isError(compressed_size), false); + std::vector pre_buffer(pre_buff_size); + std::size_t const compressed_size = + ZSTD_compress(pre_buffer.data(), pre_buff_size, in_data, in_size, compress_level); + if (ZSTD_isError(compressed_size)) + { + return false; + } - compressed_buff.append(pre_buffer, compressed_size); + compressed_buff.append(pre_buffer.data(), compressed_size); return true; } bool normal_decompress(IN const char* in_data, IN std::size_t in_size, OUT std::string& decompressed_buff) { - static char pre_buffer[PRE_ALLOCATED_ZSTD_BUFF_SIZE] = {0}; - memset(pre_buffer, 0, sizeof(pre_buffer)); - unsigned long long const r_size = ZSTD_getFrameContentSize(in_data, in_size); - ARK_ASSERT_RET_VAL(r_size != ZSTD_CONTENTSIZE_ERROR, false); - ARK_ASSERT_RET_VAL(r_size != ZSTD_CONTENTSIZE_UNKNOWN, false); + if (r_size == ZSTD_CONTENTSIZE_ERROR || r_size == ZSTD_CONTENTSIZE_UNKNOWN || + r_size > PRE_ALLOCATED_ZSTD_BUFF_SIZE) + { + return false; + } - size_t const decompressed_size = ZSTD_decompress(pre_buffer, r_size, in_data, in_size); - ARK_ASSERT_RET_VAL(!ZSTD_isError(decompressed_size), false); + std::vector pre_buffer(PRE_ALLOCATED_ZSTD_BUFF_SIZE); + size_t const decompressed_size = + ZSTD_decompress(pre_buffer.data(), static_cast(r_size), in_data, in_size); // When zstd knows the content size, it will error if it doesn't match. - ARK_ASSERT_RET_VAL(decompressed_size == r_size, false); + if (ZSTD_isError(decompressed_size) || decompressed_size != r_size) + { + return false; + } - decompressed_buff.append(pre_buffer, decompressed_size); + decompressed_buff.append(pre_buffer.data(), decompressed_size); return true; } - /// dict compress and decompress + /// Dictionary calls mutate cached contexts. Use one instance per thread or external synchronization. /** * Create zstd compress dict and decompress dict from dict file */ @@ -133,8 +147,10 @@ class zstd_compress final bool dict_compress( IN uint16_t dict_type, IN const char* in_data, IN std::size_t in_size, OUT std::string& compressed_buff) { - static char pre_buffer[PRE_ALLOCATED_ZSTD_BUFF_SIZE] = {0}; - memset(pre_buffer, 0, sizeof(pre_buffer)); + if (in_size >= PRE_ALLOCATED_ZSTD_BUFF_SIZE) + { + return false; + } size_t const pre_buff_size = ZSTD_compressBound(in_size); if (pre_buff_size >= PRE_ALLOCATED_ZSTD_BUFF_SIZE) @@ -152,22 +168,26 @@ class zstd_compress final auto iter = compress_dicts_.find(dict_type); ARK_ASSERT_RET_VAL(iter != compress_dicts_.end(), false); + std::vector pre_buffer(pre_buff_size); std::size_t const compressed_size = - ZSTD_compress_usingCDict(compress_ctx_, pre_buffer, pre_buff_size, in_data, in_size, iter->second); - ARK_ASSERT_RET_VAL(!ZSTD_isError(compressed_size), false); + ZSTD_compress_usingCDict(compress_ctx_, pre_buffer.data(), pre_buff_size, in_data, in_size, iter->second); + if (ZSTD_isError(compressed_size)) + { + return false; + } - compressed_buff.append(pre_buffer, compressed_size); + compressed_buff.append(pre_buffer.data(), compressed_size); return true; } bool dict_decompress( IN uint16_t dict_type, IN const char* in_data, IN std::size_t in_size, OUT std::string& decompressed_buff) { - static char pre_buffer[PRE_ALLOCATED_ZSTD_BUFF_SIZE] = {0}; - memset(pre_buffer, 0, sizeof(pre_buffer)); - unsigned long long const r_size = ZSTD_getFrameContentSize(in_data, in_size); - ARK_ASSERT_RET_VAL(r_size != ZSTD_CONTENTSIZE_ERROR, false); - ARK_ASSERT_RET_VAL(r_size != ZSTD_CONTENTSIZE_UNKNOWN, false); + if (r_size == ZSTD_CONTENTSIZE_ERROR || r_size == ZSTD_CONTENTSIZE_UNKNOWN || + r_size > PRE_ALLOCATED_ZSTD_BUFF_SIZE) + { + return false; + } auto iter = decompress_dicts_.find(dict_type); ARK_ASSERT_RET_VAL(iter != decompress_dicts_.end(), false); @@ -175,7 +195,10 @@ class zstd_compress final unsigned const expected_dict_id = ZSTD_getDictID_fromDDict(iter->second); unsigned const actual_dict_id = ZSTD_getDictID_fromFrame(in_data, in_size); - ARK_ASSERT_RET_VAL(actual_dict_id == expected_dict_id, false); + if (actual_dict_id != expected_dict_id) + { + return false; + } if (decompress_ctx_ == nullptr) { @@ -184,13 +207,16 @@ class zstd_compress final ARK_ASSERT_RET_VAL(decompress_ctx_ != nullptr, false); - std::size_t decompressed_size = - ZSTD_decompress_usingDDict(decompress_ctx_, pre_buffer, r_size, in_data, in_size, iter->second); - ARK_ASSERT_RET_VAL(!ZSTD_isError(decompressed_size), false); + std::vector pre_buffer(PRE_ALLOCATED_ZSTD_BUFF_SIZE); + std::size_t decompressed_size = ZSTD_decompress_usingDDict( + decompress_ctx_, pre_buffer.data(), static_cast(r_size), in_data, in_size, iter->second); // When zstd knows the content size, it will error if it doesn't match. - ARK_ASSERT_RET_VAL(decompressed_size == r_size, false); + if (ZSTD_isError(decompressed_size) || decompressed_size != r_size) + { + return false; + } - decompressed_buff.append(pre_buffer, decompressed_size); + decompressed_buff.append(pre_buffer.data(), decompressed_size); return true; } @@ -320,4 +346,4 @@ class zstd_compress final } }; -} // namespace ark \ No newline at end of file +} // namespace ark diff --git a/tests/ci/package-dependencies.test.mjs b/tests/ci/package-dependencies.test.mjs index aa5c8fd44..c4d44d354 100644 --- a/tests/ci/package-dependencies.test.mjs +++ b/tests/ci/package-dependencies.test.mjs @@ -44,13 +44,31 @@ function applicationFeatures(workflow) { test('CI configures the default etcd package feature and caches exactly those features', () => { const workflow = read('.github/workflows/ci.yml'); applicationFeatures(workflow); - assert.throws(() => applicationFeatures(workflow.replace('VCPKG_CACHE_FEATURES: \'application;boost;etcd-client\'', + assert.throws(() => applicationFeatures(workflow.replace('VCPKG_CACHE_FEATURES: \'application;boost;compression-tests;etcd-client\'', 'VCPKG_CACHE_FEATURES: \'application;boost\'')), /cache metadata/); - assert.throws(() => applicationFeatures(workflow.replaceAll('application;boost;etcd-client', 'application;boost')), /requires its package feature/); + assert.throws(() => applicationFeatures(workflow.replaceAll('application;boost;compression-tests;etcd-client', 'application;boost')), /requires its package feature/); assert.doesNotMatch(workflow, /git submodule|Initialize pinned dependencies/); assert.equal(existsSync(new URL('../../.gitmodules', import.meta.url)), false); }); +test('compression checks are opt-in and Applications CI runs all three codecs through check-core', () => { + const features = manifest.features; + assert.deepEqual(dependencies(features['compression-tests'].dependencies).sort(), ['lz4', 'zlib', 'zstd']); + assert.ok(!dependencies(features.application.dependencies).includes('lz4')); + assert.ok(!dependencies(manifest.dependencies).includes('lz4')); + assert.ok(!manifest['default-features'].includes('compression-tests')); + const root = read('CMakeLists.txt'); + assert.match(root, /option\(ARK_BUILD_COMPRESSION_TESTS[^\n]* OFF\)/); + assert.match(root, /if\(ARK_BUILD_COMPRESSION_TESTS\)[\s\S]*add_subdirectory\(tests\/compression\)[\s\S]*add_dependencies\(check-core check-compression\)/); + const checks = read('tests/compression/CMakeLists.txt'); + for (const dependency of ['ZLIB', 'Zstd', 'LZ4']) assert.match(checks, new RegExp(`find_package\\(${dependency} REQUIRED\\)`)); + assert.match(checks, /add_custom_target\(check-compression\s+COMMAND \$/); + const applications = read('.github/workflows/ci.yml').split(' applications:\n')[1].split('\n database:')[0]; + assert.match(applications, /-DVCPKG_MANIFEST_FEATURES=[^\n]*compression-tests/); + assert.match(applications, /-DARK_BUILD_COMPRESSION_TESTS=ON/); + assert.match(applications, /run: cmake --build ci-build --target check-core/); +}); + test('unsupported PhysX and simulate integrations fail explicitly instead of searching absent bundled SDKs', () => { for (const path of ['src/plugins/physic/CMakeLists.txt', 'src/server/simulate/CMakeLists.txt', 'examples/physics_scene/CMakeLists.txt']) { const source = read(path); diff --git a/tests/compression/CMakeLists.txt b/tests/compression/CMakeLists.txt new file mode 100644 index 000000000..b55a1a603 --- /dev/null +++ b/tests/compression/CMakeLists.txt @@ -0,0 +1,19 @@ +find_package(Threads REQUIRED) +find_package(ZLIB REQUIRED) +find_package(Zstd REQUIRED) +find_package(LZ4 REQUIRED) + +add_library(ark_compression_header_checks OBJECT headers/zlib.cpp headers/zstd.cpp headers/lz4.cpp) +set_property(TARGET ark_compression_header_checks PROPERTY LINK_LIBRARIES ark::core) +target_include_directories(ark_compression_header_checks PRIVATE + ${ZLIB_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIRS} ${LZ4_INCLUDE_DIR}) + +add_executable(ark_compression_tests ../core/main.cpp compression.cpp) +set_property(TARGET ark_compression_tests PROPERTY LINK_LIBRARIES + ark::core Threads::Threads ZLIB::ZLIB ${ZSTD_LIBRARIES} ${LZ4_LIBRARY}) +target_include_directories(ark_compression_tests PRIVATE ${ZSTD_INCLUDE_DIRS} ${LZ4_INCLUDE_DIR}) +add_dependencies(ark_compression_tests ark_compression_header_checks) +add_custom_target(check-compression + COMMAND $ + DEPENDS ark_compression_tests + USES_TERMINAL) diff --git a/tests/compression/compression.cpp b/tests/compression/compression.cpp new file mode 100644 index 000000000..f08d31c94 --- /dev/null +++ b/tests/compression/compression.cpp @@ -0,0 +1,292 @@ +// Copyright (c) 2026 OpenArkStudio. Licensed under Apache-2.0. See LICENSE. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../network/doctest.h" + +#include +#include +#include + +namespace { + +constexpr std::size_t capacity = 1024 * 1024; +constexpr const char* prefix = "existing:"; + +std::string payload(std::size_t size, uint32_t seed = 1) { + std::string result(size, '\0'); + for (char& byte : result) + { + seed = seed * 1664525u + 1013904223u; + byte = static_cast(seed >> 24); + } + return result; +} + +struct dictionary_file +{ + std::filesystem::path path; + + dictionary_file() + : path(std::filesystem::temp_directory_path() / + ("ark-compression-dictionary-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()))) { + std::ofstream output(path, std::ios::binary); + const auto bytes = payload(4096); + output.write(bytes.data(), static_cast(bytes.size())); + output.close(); + if (!output) + { + throw std::runtime_error("cannot write temporary dictionary"); + } + } + + ~dictionary_file() { + std::error_code error; + std::filesystem::remove(path, error); + } +}; + +std::string external_zlib(const std::string& input) { + uLongf size = compressBound(static_cast(input.size())); + std::string output(size, '\0'); + REQUIRE(compress2(reinterpret_cast(output.data()), &size, reinterpret_cast(input.data()), + static_cast(input.size()), 1) == Z_OK); + output.resize(size); + return output; +} + +std::string external_lz4(const std::string& input) { + std::string output(LZ4_compressBound(static_cast(input.size())), '\0'); + const auto size = LZ4_compress_default( + input.data(), output.data(), static_cast(input.size()), static_cast(output.size())); + REQUIRE(size > 0); + output.resize(size); + return output; +} + +std::string external_zstd(const std::string& input, bool content_size = true) { + auto* context = ZSTD_createCCtx(); + REQUIRE(context != nullptr); + REQUIRE_FALSE(ZSTD_isError(ZSTD_CCtx_setParameter(context, ZSTD_c_contentSizeFlag, content_size ? 1 : 0))); + std::string output(ZSTD_compressBound(input.size()), '\0'); + const auto size = ZSTD_compress2(context, output.data(), output.size(), input.data(), input.size()); + ZSTD_freeCCtx(context); + REQUIRE_FALSE(ZSTD_isError(size)); + output.resize(size); + return output; +} + +template +void check_compression_boundary(bound_type bound, compress_type compress) { + std::size_t rejected = 0; + while (static_cast(bound(rejected)) < capacity) + { + ++rejected; + } + REQUIRE(rejected > 0); + auto input = payload(rejected); + std::string output = prefix; + CHECK(compress(input.data(), rejected - 1, output)); + CHECK(output.starts_with(prefix)); + output = prefix; + CHECK_FALSE(compress(input.data(), rejected, output)); + CHECK(output == prefix); + CHECK_FALSE(compress(input.data(), (std::numeric_limits::max)(), output)); + CHECK(output == prefix); +} + +} // namespace + +TEST_CASE("compression preserves binary bytes and appends successful results") { + const auto input = payload(8192); + ark::zstd_compress zstd; + for (int algorithm = 0; algorithm < 3; ++algorithm) + { + std::string compressed = prefix; + const bool packed = algorithm == 0 ? ark::zlib_compress::compress(input.data(), input.size(), 1, compressed) + : algorithm == 1 ? ark::lz4_compress::compress(input.data(), input.size(), 1, compressed) + : zstd.normal_compress(input.data(), input.size(), 1, compressed); + REQUIRE(packed); + REQUIRE(compressed.starts_with(prefix)); + compressed.erase(0, std::char_traits::length(prefix)); + std::string output = prefix; + const bool unpacked = + algorithm == 0 ? ark::zlib_compress::decompress(compressed.data(), compressed.size(), output) + : algorithm == 1 ? ark::lz4_compress::decompress(compressed.data(), compressed.size(), output) + : zstd.normal_decompress(compressed.data(), compressed.size(), output); + CHECK(unpacked); + CHECK(output == prefix + input); + } +} + +TEST_CASE("compression retains strict compression bound and rejects unrepresentable input sizes") { + check_compression_boundary([](std::size_t size) { return compressBound(static_cast(size)); }, + [](const char* data, std::size_t size, std::string& output) { + return ark::zlib_compress::compress(data, size, 1, output); + }); + check_compression_boundary([](std::size_t size) { return LZ4_compressBound(static_cast(size)); }, + [](const char* data, std::size_t size, std::string& output) { + return ark::lz4_compress::compress(data, size, 1, output); + }); + ark::zstd_compress zstd; + check_compression_boundary([](std::size_t size) { return ZSTD_compressBound(size); }, + [&](const char* data, std::size_t size, std::string& output) { + return zstd.normal_compress(data, size, 1, output); + }); + dictionary_file dictionary; + REQUIRE(zstd.init_z_std_dict(1, dictionary.path.string(), 1)); + check_compression_boundary([](std::size_t size) { return ZSTD_compressBound(size); }, + [&](const char* data, std::size_t size, std::string& output) { + return zstd.dict_compress(1, data, size, output); + }); + + std::string output = prefix; + const auto too_large = static_cast((std::numeric_limits::max)()) + 1; + CHECK_FALSE(ark::lz4_compress::compress("x", static_cast(LZ4_MAX_INPUT_SIZE) + 1, 1, output)); + CHECK_FALSE(ark::lz4_compress::decompress("x", too_large, output)); + if constexpr (sizeof(std::size_t) > sizeof(uLong)) + { + const auto source_size = static_cast((std::numeric_limits::max)()) + 1; + CHECK_FALSE(ark::zlib_compress::decompress("x", source_size, output)); + } + CHECK(output == prefix); +} + +TEST_CASE("decompression accepts exactly one MiB and rejects larger outputs") { + ark::zstd_compress zstd; + dictionary_file dictionary; + REQUIRE(zstd.init_z_std_dict(1, dictionary.path.string(), 1)); + for (const std::size_t size : {capacity - 1, capacity, capacity + 1}) + { + const std::string input(size, 'a'); + const auto zlib_frame = external_zlib(input); + const auto lz4_frame = external_lz4(input); + const auto zstd_frame = external_zstd(input); + std::string output = prefix; + CHECK(ark::zlib_compress::decompress(zlib_frame.data(), zlib_frame.size(), output) == (size <= capacity)); + CHECK(output == (size <= capacity ? prefix + input : prefix)); + output = prefix; + CHECK(ark::lz4_compress::decompress(lz4_frame.data(), lz4_frame.size(), output) == (size <= capacity)); + CHECK(output == (size <= capacity ? prefix + input : prefix)); + output = prefix; + CHECK(zstd.normal_decompress(zstd_frame.data(), zstd_frame.size(), output) == (size <= capacity)); + CHECK(output == (size <= capacity ? prefix + input : prefix)); + // A raw-content dictionary has ID zero, matching this standard frame. + output = prefix; + CHECK(zstd.dict_decompress(1, zstd_frame.data(), zstd_frame.size(), output) == (size <= capacity)); + CHECK(output == (size <= capacity ? prefix + input : prefix)); + } +} + +TEST_CASE("corrupt truncated and unknown-size frames preserve caller output") { + ark::zstd_compress zstd; + dictionary_file dictionary; + REQUIRE(zstd.init_z_std_dict(1, dictionary.path.string(), 1)); + const std::string input = payload(512); + const auto zlib_frame = external_zlib(input); + const auto lz4_frame = external_lz4(input); + const auto zstd_frame = external_zstd(input); + std::string output = prefix; + CHECK_FALSE(ark::zlib_compress::decompress(zlib_frame.data(), zlib_frame.size() - 1, output)); + CHECK_FALSE(ark::lz4_compress::decompress(lz4_frame.data(), lz4_frame.size() - 1, output)); + CHECK_FALSE(zstd.normal_decompress(zstd_frame.data(), zstd_frame.size() - 1, output)); + CHECK_FALSE(zstd.dict_decompress(1, zstd_frame.data(), zstd_frame.size() - 1, output)); + CHECK_FALSE(ark::zlib_compress::decompress("bad", 3, output)); + CHECK_FALSE(ark::lz4_compress::decompress("\xff", 1, output)); + const std::string different_dictionary("\x28\xb5\x2f\xfd\x21\x01\x00\x01\x00\x00", 10); + REQUIRE(ZSTD_getFrameContentSize(different_dictionary.data(), different_dictionary.size()) == 0); + REQUIRE(ZSTD_getDictID_fromFrame(different_dictionary.data(), different_dictionary.size()) == 1); + CHECK_FALSE(zstd.dict_decompress(1, different_dictionary.data(), different_dictionary.size(), output)); + for (const auto& frame : {std::string("bad"), external_zstd(input, false), + std::string("\x28\xb5\x2f\xfd\xe0\x01\x00\x00\x00\x01\x00\x00\x00", 13)}) + { + CHECK_FALSE(zstd.normal_decompress(frame.data(), frame.size(), output)); + CHECK_FALSE(zstd.dict_decompress(1, frame.data(), frame.size(), output)); + } + CHECK(output == prefix); +} + +TEST_CASE("empty inputs retain each codec's existing behavior") { + ark::zstd_compress zstd; + dictionary_file dictionary; + REQUIRE(zstd.init_z_std_dict(1, dictionary.path.string(), 1)); + std::string compressed; + std::string output = prefix; + REQUIRE(ark::zlib_compress::compress("", 0, 1, compressed)); + CHECK(ark::zlib_compress::decompress(compressed.data(), compressed.size(), output)); + compressed.clear(); + REQUIRE(ark::lz4_compress::compress("", 0, 1, compressed)); + CHECK_FALSE(ark::lz4_compress::decompress(compressed.data(), compressed.size(), output)); + compressed.clear(); + REQUIRE(zstd.normal_compress("", 0, 1, compressed)); + CHECK(zstd.normal_decompress(compressed.data(), compressed.size(), output)); + compressed.clear(); + REQUIRE(zstd.dict_compress(1, "", 0, compressed)); + CHECK(zstd.dict_decompress(1, compressed.data(), compressed.size(), output)); + CHECK(output == prefix); +} + +TEST_CASE("parallel compression calls own their output buffers") { + dictionary_file dictionary; + ark::zstd_compress normal_zstd; + constexpr int workers = 6; + std::barrier start(workers); + std::atomic passed = true; + std::vector threads; + for (int worker = 0; worker < workers; ++worker) + { + threads.emplace_back([&, worker] { + // Dictionary contexts belong to one thread; sharing one instance is unsupported. + ark::zstd_compress zstd; + if (!zstd.init_z_std_dict(1, dictionary.path.string(), 1)) + { + passed = false; + } + start.arrive_and_wait(); + for (int iteration = 0; iteration < 20; ++iteration) + { + const auto input = payload(128 * 1024 + iteration, worker * 100 + iteration + 1); + std::string packed; + std::string output = prefix; + if (!ark::zlib_compress::compress(input.data(), input.size(), 1, packed) || + !ark::zlib_compress::decompress(packed.data(), packed.size(), output) || output != prefix + input) + { + passed = false; + } + packed.clear(); + output = prefix; + if (!ark::lz4_compress::compress(input.data(), input.size(), 1, packed) || + !ark::lz4_compress::decompress(packed.data(), packed.size(), output) || output != prefix + input) + { + passed = false; + } + packed.clear(); + output = prefix; + if (!normal_zstd.normal_compress(input.data(), input.size(), 1, packed) || + !normal_zstd.normal_decompress(packed.data(), packed.size(), output) || output != prefix + input) + { + passed = false; + } + packed.clear(); + output = prefix; + if (!zstd.dict_compress(1, input.data(), input.size(), packed) || + !zstd.dict_decompress(1, packed.data(), packed.size(), output) || output != prefix + input) + { + passed = false; + } + } + }); + } + threads.clear(); + CHECK(passed.load()); +} diff --git a/tests/compression/headers/lz4.cpp b/tests/compression/headers/lz4.cpp new file mode 100644 index 000000000..8b4afeca6 --- /dev/null +++ b/tests/compression/headers/lz4.cpp @@ -0,0 +1,2 @@ +// Copyright (c) 2026 OpenArkStudio. Licensed under Apache-2.0. See LICENSE. +#include diff --git a/tests/compression/headers/zlib.cpp b/tests/compression/headers/zlib.cpp new file mode 100644 index 000000000..d06c73236 --- /dev/null +++ b/tests/compression/headers/zlib.cpp @@ -0,0 +1,2 @@ +// Copyright (c) 2026 OpenArkStudio. Licensed under Apache-2.0. See LICENSE. +#include diff --git a/tests/compression/headers/zstd.cpp b/tests/compression/headers/zstd.cpp new file mode 100644 index 000000000..249bcfb62 --- /dev/null +++ b/tests/compression/headers/zstd.cpp @@ -0,0 +1,2 @@ +// Copyright (c) 2026 OpenArkStudio. Licensed under Apache-2.0. See LICENSE. +#include diff --git a/tests/core/README.md b/tests/core/README.md index a6a739a94..94409c9f1 100644 --- a/tests/core/README.md +++ b/tests/core/README.md @@ -55,6 +55,23 @@ in a dot keep that basename. All test writes use temporary directories. cmake --build core-build --target check-logging ``` +## Compression helpers + +`ARK_BUILD_COMPRESSION_TESTS=ON` adds `check-compression` to `check-core` and +also supports running it independently. Install the vcpkg `compression-tests` +feature for zlib, Zstandard and LZ4; the option defaults to `OFF` so core-only +builds keep their existing dependency requirements. Applications CI explicitly +enables it. Tests compile each mapped `` header independently +and cover binary/empty inputs, append semantics, strict compression bounds, +exactly 1 MiB decompression, oversized/unknown/corrupt frames, integer-size +rejection and concurrent round trips. Dictionary workers use separate instances. +See [Utility compression](../../docs/reference/utility.md#compression-buffers-and-concurrency) +for the caller-owned output and dictionary-context synchronization requirements. + +```sh +cmake --build core-build --target check-compression --parallel 2 +``` + ## Public task interface `check-task-sdk` is included in `check-core`. It compiles `` diff --git a/tests/core/README_CN.md b/tests/core/README_CN.md index a6d7c83b6..b351bc344 100644 --- a/tests/core/README_CN.md +++ b/tests/core/README_CN.md @@ -47,6 +47,20 @@ C++20 语言回归验证六种精确整数类型分类、限定符与引用处 cmake --build core-build --target check-logging ``` +## 压缩辅助函数 + +`ARK_BUILD_COMPRESSION_TESTS=ON` 将 `check-compression` 加入 `check-core`，也支持单独 +运行此目标。通过 vcpkg 的 `compression-tests` feature 安装 zlib、Zstandard 和 LZ4。 +该选项默认为 `OFF`，保留仅构建核心时的既有依赖要求；Applications CI 会显式开启。 +检查会独立编译每个映射后的 `` 头，覆盖二进制和空输入、追加语义、 +严格的压缩大小边界、恰好 1 MiB 的解压、超限或未知大小帧、损坏帧、整数长度拒绝和 +并发往返。字典工作线程使用独立实例。调用方输出字符串和字典上下文的同步要求见 +[Utility 压缩说明](../../docs/reference/utility_CN.md#压缩缓冲区与并发)。 + +```sh +cmake --build core-build --target check-compression --parallel 2 +``` + ## 公共任务接口 `check-core` 包含 `check-task-sdk`：独立编译仅包含 `` 的文件， diff --git a/vcpkg.json b/vcpkg.json index 8f46e9460..1c1a9d666 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -11,6 +11,14 @@ "readerwriterqueue" ], "features": { + "compression-tests": { + "description": "Build focused compression regression tests", + "dependencies": [ + "lz4", + "zlib", + "zstd" + ] + }, "database": { "description": "Build native database development tests with OpenSSL authentication and TLS", "dependencies": [ From 4226019466960ac036598437771030f7ecb392dd Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 23 Oct 2026 13:59:13 +0800 Subject: [PATCH 3/6] fix(crypto): bound XXTEA key normalization --- docs/reference/xxtea.md | 20 ++++++++++++ docs/reference/xxtea_CN.md | 20 ++++++++++++ docs/site/src/navigation.ts | 5 +++ src/core/crypto/xxtea.hpp | 27 ++++++++-------- tests/core/CMakeLists.txt | 2 +- tests/core/README.md | 3 ++ tests/core/README_CN.md | 2 ++ tests/core/xxtea.cpp | 64 +++++++++++++++++++++++++++++++++++++ 8 files changed, 129 insertions(+), 14 deletions(-) create mode 100644 docs/reference/xxtea.md create mode 100644 docs/reference/xxtea_CN.md create mode 100644 tests/core/xxtea.cpp diff --git a/docs/reference/xxtea.md b/docs/reference/xxtea.md new file mode 100644 index 000000000..62aab742d --- /dev/null +++ b/docs/reference/xxtea.md @@ -0,0 +1,20 @@ +# XXTEA helper + +`` provides the existing XXTEA format through the `ark::core` CMake target. No runtime plugin is required. + +```cpp +#include +#include + +std::string ciphertext; +std::string plaintext; +if (ark::xxtea::xxtea_encrypt("message", "key", ciphertext)) { + const bool decoded = ark::xxtea::xxtea_decrypt(ciphertext, "key", plaintext); +} +``` + +The string overloads append to the output on success and return `false` for empty input. Keys use at most 16 bytes: the first zero byte terminates a short key, and the remaining bytes are zero-filled. A key of exactly 16 nonzero bytes does not require a terminator. Longer keys use only their first 16 bytes. This normalization preserves the existing ciphertext format. + +The raw overloads accept a non-null key pointing to either a terminated short key or at least 16 readable bytes. They return allocated storage, its length through `out_len`, and require the caller to release successful results with `std::free()`. Prefer the string overloads for owning results. Do not pass a null key, input pointer or length pointer to the raw interface. + +`check-core` covers exact-size short-key allocations, full-width and embedded-zero keys, binary round trips and output appending. AddressSanitizer also checks that normalization does not read beyond a short key's terminator. diff --git a/docs/reference/xxtea_CN.md b/docs/reference/xxtea_CN.md new file mode 100644 index 000000000..8f4630463 --- /dev/null +++ b/docs/reference/xxtea_CN.md @@ -0,0 +1,20 @@ +# XXTEA 工具 + +`` 通过 `ark::core` CMake 目标提供现有 XXTEA 格式，无需加载运行时插件。 + +```cpp +#include +#include + +std::string ciphertext; +std::string plaintext; +if (ark::xxtea::xxtea_encrypt("message", "key", ciphertext)) { + const bool decoded = ark::xxtea::xxtea_decrypt(ciphertext, "key", plaintext); +} +``` + +字符串重载成功时向输出追加结果，输入为空时返回 `false`。密钥最多使用 16 字节：短密钥在首个零字节处结束，剩余位置补零；恰好包含 16 个非零字节的密钥无需终止符；更长的密钥只使用前 16 字节。此规则保留现有密文格式。 + +原始指针重载要求密钥非空，指向带终止符的短密钥，或至少 16 个可读字节。成功时返回分配的内存，通过 `out_len` 返回长度，调用方使用 `std::free()` 释放结果。需要拥有型结果时优先使用字符串重载。原始指针接口的密钥、输入数据与长度指针均不能传入空指针。 + +`check-core` 覆盖精确长度分配的短密钥、完整 16 字节密钥、内嵌零字节、二进制往返及输出追加。AddressSanitizer 同时检查密钥处理是否读过短密钥的终止位置。 diff --git a/docs/site/src/navigation.ts b/docs/site/src/navigation.ts index 021783415..2949f2b9b 100644 --- a/docs/site/src/navigation.ts +++ b/docs/site/src/navigation.ts @@ -135,6 +135,11 @@ const tree: Entry[] = [ "variant": "info" }, "children": [ + [ + "reference/xxtea", + "XXTEA helper", + "XXTEA 工具" + ], [ "reference/object-vectors", "Object vectors", diff --git a/src/core/crypto/xxtea.hpp b/src/core/crypto/xxtea.hpp index 90d5fac0d..eef9712f6 100644 --- a/src/core/crypto/xxtea.hpp +++ b/src/core/crypto/xxtea.hpp @@ -25,15 +25,6 @@ #define MX (((z >> 5) ^ (y << 2)) + ((y >> 3) ^ (z << 4))) ^ ((sum ^ y) + (key[(p & 3) ^ e] ^ z)) #define DELTA 0x9e3779b9 -#define FIXED_KEY \ - size_t i; \ - uint8_t fixed_key[16]; \ - memcpy(fixed_key, key, 16); \ - for (i = 0; (i < 16) && (fixed_key[i] != 0); ++i) \ - ; \ - for (++i; i < 16; ++i) \ - fixed_key[i] = 0; - namespace ark { class xxtea @@ -90,8 +81,8 @@ class xxtea * Caller is responsible for freeing the returned buffer. */ static void* xxtea_encrypt(const void* data, size_t len, const void* key, size_t* out_len) { - FIXED_KEY - return xxtea_ubyte_encrypt((const uint8_t*)data, len, fixed_key, out_len); + const auto fixed_key = normalize_key(key); + return xxtea_ubyte_encrypt((const uint8_t*)data, len, fixed_key.data(), out_len); } /** @@ -105,11 +96,21 @@ class xxtea * Caller is responsible for freeing the returned buffer. */ static void* xxtea_decrypt(const void* data, size_t len, const void* key, size_t* out_len) { - FIXED_KEY - return xxtea_ubyte_decrypt((const uint8_t*)data, len, fixed_key, out_len); + const auto fixed_key = normalize_key(key); + return xxtea_ubyte_decrypt((const uint8_t*)data, len, fixed_key.data(), out_len); } protected: + static std::array normalize_key(const void* key) { + std::array result{}; + const auto* bytes = static_cast(key); + for (size_t index = 0; index < result.size() && bytes[index] != 0; ++index) + { + result[index] = bytes[index]; + } + return result; + } + static uint32_t* xxtea_to_uint_array(const uint8_t* data, size_t len, int inc_len, size_t* out_len) { uint32_t* out; size_t n; diff --git a/tests/core/CMakeLists.txt b/tests/core/CMakeLists.txt index 3999aa898..5a97b76f4 100644 --- a/tests/core/CMakeLists.txt +++ b/tests/core/CMakeLists.txt @@ -6,7 +6,7 @@ target_link_libraries(ark_core_header_checks PRIVATE ark::core) add_executable(ark_core_tests main.cpp strings.cpp containers.cpp templates.cpp callbacks.cpp header_regressions.cpp cxx20_language.cpp concurrency.cpp plugin_manager_startup.cpp data_strings.cpp node_data_storage.cpp - object_vectors.cpp plugin_construction.cpp module_factory.cpp) + object_vectors.cpp plugin_construction.cpp module_factory.cpp xxtea.cpp) target_link_libraries(ark_core_tests PRIVATE ark::core ark::kernel ark::utility) add_dependencies(ark_core_tests ark_core_header_checks) add_library(ark_construction_header_checks OBJECT headers/module_base.cpp headers/plugin_base.cpp) diff --git a/tests/core/README.md b/tests/core/README.md index a6a739a94..447f99670 100644 --- a/tests/core/README.md +++ b/tests/core/README.md @@ -114,3 +114,6 @@ No external HTTP/etcd request is made. See [HTTP examples](../../docs/reference/ for the preserved fixed demonstration endpoints and the real-wire coverage boundary. With Protobuf and the Kernel/Utility runtime targets enabled, `check-kernel-plugin` loads their actual DSOs through an independent public-interface runner. It checks the four original interface keys and manager pointers, metadata/static record loading, map/entity registration and callbacks, manager updates, shutdown destruction and repeated stop. It is part of `check-core`, alongside the existing direct initialization/entity-default tests; no post-unload registry inspection or arbitrary hot-update safety is claimed. The Kernel public-header consumer checks all retained types and their typed bases/update declarations. + +The XXTEA cases cover exact-size short-key allocations, 16-byte and embedded-zero +keys, binary round trips and output appending. See the [key contract](../../docs/reference/xxtea.md). diff --git a/tests/core/README_CN.md b/tests/core/README_CN.md index a6d7c83b6..6a0a33c79 100644 --- a/tests/core/README_CN.md +++ b/tests/core/README_CN.md @@ -99,3 +99,5 @@ Net 注册的 HTTP 客户端现在由私有接口工厂创建。`check-net-plugi 保留的固定演示端点与真实传输验证边界见 [HTTP 示例](../../docs/reference/http_CN.md#旧-http-示例的类型化创建)。 开启 Protobuf 且存在 Kernel／Utility 运行时 target 时，`check-kernel-plugin` 通过独立公开接口 runner 加载真实 DSO，检查原有四个接口键与管理器指针、元数据／静态记录加载、地图／实体注册和回调、管理器更新、停机销毁与重复 stop。它与现有直接初始化／实体默认值测试一起纳入 `check-core`；不在卸载后检查注册表，也不证明任意热更新安全。Kernel 公开头消费者检查全部保留类型的基类和 update 声明。 + +XXTEA 用例覆盖精确长度分配的短密钥、16 字节与内嵌零字节密钥、二进制往返及输出追加，参见[密钥约定](../../docs/reference/xxtea_CN.md)。 diff --git a/tests/core/xxtea.cpp b/tests/core/xxtea.cpp new file mode 100644 index 000000000..22d5472ce --- /dev/null +++ b/tests/core/xxtea.cpp @@ -0,0 +1,64 @@ +// Copyright (c) 2026 OpenArkStudio. Licensed under Apache-2.0. See LICENSE. +#include +#include +#include +#include +#include + +#include "../network/doctest.h" + +#include + +namespace { + +std::string encrypt(const std::string& message, const void* key) { + size_t length = 0; + std::unique_ptr value( + ark::xxtea::xxtea_encrypt(message.data(), message.size(), key, &length), &std::free); + if (!value) + { + return {}; + } + return {static_cast(value.get()), length}; +} + +} // namespace + +DOCTEST_TEST_CASE("XXTEA reads short keys only through their terminating zero") { + const std::string message("binary\0message", 14); + for (size_t length = 0; length < 16; ++length) + { + DOCTEST_CAPTURE(length); + // Exact allocation makes reading beyond the terminator observable. + auto short_key = std::make_unique(length + 1); + std::fill_n(short_key.get(), length, 'k'); + short_key[length] = '\0'; + std::array padded{}; + std::fill_n(padded.begin(), length, 'k'); + const auto ciphertext = encrypt(message, short_key.get()); + DOCTEST_REQUIRE_FALSE(ciphertext.empty()); + DOCTEST_CHECK(ciphertext == encrypt(message, padded.data())); + size_t decoded_size = 0; + std::unique_ptr decoded( + ark::xxtea::xxtea_decrypt(ciphertext.data(), ciphertext.size(), short_key.get(), &decoded_size), + &std::free); + DOCTEST_REQUIRE(decoded != nullptr); + DOCTEST_CHECK(std::string(static_cast(decoded.get()), decoded_size) == message); + } +} + +DOCTEST_TEST_CASE("XXTEA preserves full-width and zero-terminated key normalization") { + const std::string message = "message"; + const std::array full{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + DOCTEST_CHECK(encrypt(message, full.data()) == encrypt(message, "0123456789abcdefignored")); + const std::array embedded{'k', '\0', 'x', 'y'}; + DOCTEST_CHECK(encrypt(message, embedded.data()) == encrypt(message, "k")); + std::string ciphertext = "prefix:"; + DOCTEST_REQUIRE(ark::xxtea::xxtea_encrypt(message, "k", ciphertext)); + DOCTEST_CHECK(ciphertext == "prefix:" + encrypt(message, "k")); + std::string plaintext = "prefix:"; + DOCTEST_REQUIRE(ark::xxtea::xxtea_decrypt(ciphertext.substr(7), "k", plaintext)); + DOCTEST_CHECK(plaintext == "prefix:" + message); + DOCTEST_CHECK_FALSE(ark::xxtea::xxtea_encrypt(std::string{}, "k", ciphertext)); + DOCTEST_CHECK_FALSE(ark::xxtea::xxtea_decrypt(std::string{}, "k", plaintext)); +} From 41c392ed1feaff9aa71c3cfd0f910d570c309746 Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 23 Oct 2026 14:48:55 +0800 Subject: [PATCH 4/6] ci(cache): reuse native compiler objects on supported runners --- .github/scripts/cache-stats.mjs | 29 +++++ .github/scripts/compiler-cache.mjs | 54 +++++++++ .github/workflows/ci.yml | 146 ++++++++++++++++++++++++- AGENTS.md | 6 +- docs/project/contributing.md | 51 ++++++++- docs/project/contributing_CN.md | 37 ++++++- tests/ci/cache-workflow.test.mjs | 45 +++++++- tests/ci/compiler-cache.test.mjs | 170 +++++++++++++++++++++++++++++ 8 files changed, 530 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/cache-stats.mjs create mode 100644 .github/scripts/compiler-cache.mjs create mode 100644 tests/ci/compiler-cache.test.mjs diff --git a/.github/scripts/cache-stats.mjs b/.github/scripts/cache-stats.mjs new file mode 100644 index 000000000..3c1539331 --- /dev/null +++ b/.github/scripts/cache-stats.mjs @@ -0,0 +1,29 @@ +import { spawnSync } from 'node:child_process'; +import { appendFileSync, readdirSync } from 'node:fs'; + +let archives = 'unavailable'; +if (process.env.VCPKG_ARCHIVE_DIRECTORY) { + try { + archives = readdirSync(process.env.VCPKG_ARCHIVE_DIRECTORY, { recursive: true }) + .filter(file => file.endsWith('.zip')).length; + } catch { + // Cache diagnostics do not change the build or test result. + } +} +const lines = [ + `Dependency archive cache exact hit: ${process.env.VCPKG_ARCHIVE_HIT || 'false/unavailable'}`, + `Dependency archives after configure/build: ${archives}`, + 'vcpkg package restore/build counts are in the Configure step output.', + `Compiler archive cache exact hit: ${process.env.COMPILER_ARCHIVE_HIT || 'false/unavailable'} (a prefix restore can still provide compiler hits)`, +]; +if (process.env.ARK_COMPILER_LAUNCHER) { + const result = spawnSync(process.env.ARK_COMPILER_LAUNCHER, ['--show-stats'], { + encoding: 'utf8', timeout: 30_000, + }); + lines.push(result.status === 0 ? result.stdout.trim() : 'sccache statistics unavailable'); +} else { + lines.push('Compiler cache disabled; compilation uses the compiler directly.'); +} +const report = lines.join('\n'); +console.log(report); +if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `\n### Native build cache\n\n\`\`\`text\n${report}\n\`\`\`\n`); diff --git a/.github/scripts/compiler-cache.mjs b/.github/scripts/compiler-cache.mjs new file mode 100644 index 000000000..180cff5b1 --- /dev/null +++ b/.github/scripts/compiler-cache.mjs @@ -0,0 +1,54 @@ +import { spawnSync } from 'node:child_process'; +import { appendFileSync, lstatSync, mkdirSync } from 'node:fs'; +import { posix, resolve, win32 } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +export const sccacheVersion = '0.16.0'; + +export function prepareCompilerCache({ env, run = spawnSync, start = true }) { + if (!env.GITHUB_ENV || !env.GITHUB_OUTPUT) throw new Error('GitHub file-command channels are required'); + // The dependency step already fingerprints the runner, profile and dependency inputs. + const identity = env.COMPILER_CACHE_IDENTITY ?? ''; + const validIdentity = /^ark-vcpkg-files-v2-(Linux|macOS|Windows)-(X64|ARM64|ARM)-[a-f0-9]{64}-[a-f0-9]{64}$/.test(identity) && + identity.startsWith(`ark-vcpkg-files-v2-${env.RUNNER_OS}-${env.RUNNER_ARCH}-`); + const validSource = /^[a-f0-9]{40}$/.test(env.GITHUB_SHA ?? ''); + const paths = env.RUNNER_OS === 'Windows' ? win32 : posix; + const compatible = validIdentity && validSource && env.RUNNER_OS !== 'Windows' && + paths.isAbsolute(env.RUNNER_TEMP ?? '') && !/[\x00-\x1f\x7f;,`*?\[\]{}!]/.test(env.RUNNER_TEMP ?? ''); + const directory = compatible ? paths.join(env.RUNNER_TEMP, 'ark-sccache') : ''; + const key = compatible ? identity.replace('vcpkg-files-v2', `sccache-${sccacheVersion}-v1`) : ''; + // Immutable snapshots can grow across source revisions without invalidating object compatibility. + const selection = { archiveKey: key ? `${key}-${env.GITHUB_SHA}` : '', restoreKeys: key ? `${key}-` : '' }; + let enabled = false; + if (directory && (!start || env.SCCACHE_RESTORE_OUTCOME === 'success') && + env.SCCACHE_SETUP_OUTCOME === 'success' && env.SCCACHE_PATH && + paths.isAbsolute(env.SCCACHE_PATH) && !/[\x00-\x1f\x7f;]/.test(env.SCCACHE_PATH)) { + try { + mkdirSync(directory, { recursive: true }); + if (!lstatSync(directory).isDirectory()) throw new Error('cache path is not a directory'); + const cacheEnv = { ...env, SCCACHE_DIR: directory, SCCACHE_GHA_ENABLED: 'false', + SCCACHE_IGNORE_SERVER_IO_ERROR: '1', SCCACHE_CACHE_SIZE: '512M', SCCACHE_C_CUSTOM_CACHE_BUSTER: key }; + const options = { env: cacheEnv, encoding: 'utf8', timeout: 30_000 }; + const version = run(env.SCCACHE_PATH, ['--version'], options); + if (version.status === 0 && version.stdout.trim() === `sccache ${sccacheVersion}`) { + enabled = !start || run(env.SCCACHE_PATH, ['--start-server'], options).status === 0; + } + } catch { + // Optional cache setup must not prevent a normal compiler invocation. + } + } + const launcher = enabled ? env.SCCACHE_PATH.replaceAll('\\', '/') : ''; + appendFileSync(env.GITHUB_ENV, `ARK_COMPILER_LAUNCHER=${launcher}\nSCCACHE_DIR=${enabled ? directory : ''}\nSCCACHE_GHA_ENABLED=false\nSCCACHE_IGNORE_SERVER_IO_ERROR=1\nSCCACHE_CACHE_SIZE=512M\nSCCACHE_C_CUSTOM_CACHE_BUSTER=${key}\n`); + appendFileSync(env.GITHUB_OUTPUT, `enabled=${enabled}\ndirectory=${enabled ? directory : ''}\nkey=${enabled ? key : ''}\narchive_key=${enabled ? selection.archiveKey : ''}\nrestore_keys=${enabled ? selection.restoreKeys : ''}\n`); + return { enabled, directory, key, launcher, ...selection }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + const plan = prepareCompilerCache({ env: process.env, start: !process.argv.includes('--prepare') }); + console.log(plan.enabled ? `sccache ${sccacheVersion} compiler launcher prepared` : 'Compiler cache unavailable; using the compiler directly'); + } catch { + console.error('Unable to export compiler cache configuration to GitHub file-command channels'); + process.exitCode = 1; + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5197e1c9e..6c3800006 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,12 +122,43 @@ jobs: path: ${{ steps.vcpkg-cache.outputs.directory }} key: ${{ steps.vcpkg-cache.outputs.key }} restore-keys: ${{ steps.vcpkg-cache.outputs.restore-key }} + - name: Install compiler cache + id: sccache-setup + if: runner.os != 'Windows' + continue-on-error: true + uses: Mozilla-Actions/sccache-action@v0.0.11 + with: + version: v0.16.0 + disable_annotations: true + - name: Prepare compiler launcher + id: compiler-cache + env: + COMPILER_CACHE_IDENTITY: ${{ steps.vcpkg-cache.outputs.key }} + SCCACHE_SETUP_OUTCOME: ${{ steps.sccache-setup.outcome }} + run: node .github/scripts/compiler-cache.mjs --prepare + - name: Restore compiler cache + id: compiler-restore + if: steps.compiler-cache.outputs.enabled == 'true' + continue-on-error: true + uses: actions/cache/restore@v4 + with: + path: ${{ steps.compiler-cache.outputs.directory }} + key: ${{ steps.compiler-cache.outputs.archive_key }} + restore-keys: ${{ steps.compiler-cache.outputs.restore_keys }} + - name: Start compiler cache after archive restore + id: compiler-start + if: steps.compiler-cache.outputs.enabled == 'true' + env: + COMPILER_CACHE_IDENTITY: ${{ steps.vcpkg-cache.outputs.key }} + SCCACHE_SETUP_OUTCOME: ${{ steps.sccache-setup.outcome }} + SCCACHE_RESTORE_OUTCOME: ${{ steps.compiler-restore.outcome }} + run: node .github/scripts/compiler-cache.mjs - uses: lukka/get-cmake@latest - uses: lukka/run-vcpkg@v11 with: vcpkgGitCommitId: 0699a19d0c6386247ce50d4dedbb8217d484d536 vcpkgJsonGlob: vcpkg.json - - name: Configure TSan address space + - name: Configure ThreadSanitizer address space if: matrix.sanitizer == 'thread' run: sudo sysctl -w vm.mmap_rnd_bits=28 - name: Configure network regressions @@ -137,6 +168,8 @@ jobs: "-DCMAKE_TOOLCHAIN_FILE=$env:VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" -DVCPKG_MANIFEST_FEATURES=boost -DCMAKE_BUILD_TYPE=Debug + "-DCMAKE_C_COMPILER_LAUNCHER=$env:ARK_COMPILER_LAUNCHER" + "-DCMAKE_CXX_COMPILER_LAUNCHER=$env:ARK_COMPILER_LAUNCHER" -DARK_BUILD_APPLICATIONS=OFF -DARK_BUILD_NETWORK_TESTS=ON -DASIO_STANDALONE=OFF @@ -155,6 +188,21 @@ jobs: UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 TSAN_OPTIONS: halt_on_error=1:second_deadlock_stack=1 run: cmake --build ci-build --target check-network + - name: Report native cache statistics + if: ${{ !cancelled() }} + continue-on-error: true + env: + VCPKG_ARCHIVE_DIRECTORY: ${{ steps.vcpkg-cache.outputs.directory }} + VCPKG_ARCHIVE_HIT: ${{ steps.vcpkg-restore.outputs.cache-hit }} + COMPILER_ARCHIVE_HIT: ${{ steps.compiler-restore.outputs.cache-hit }} + run: node .github/scripts/cache-stats.mjs + - name: Save compiler cache + if: ${{ success() && steps.compiler-start.outputs.enabled == 'true' && steps.compiler-restore.outputs.cache-hit != 'true' }} + continue-on-error: true + uses: actions/cache/save@v4 + with: + path: ${{ steps.compiler-cache.outputs.directory }} + key: ${{ steps.compiler-cache.outputs.archive_key }} applications: name: Applications - ${{ matrix.os }} - C++20 needs: scope @@ -188,6 +236,37 @@ jobs: path: ${{ steps.vcpkg-cache.outputs.directory }} key: ${{ steps.vcpkg-cache.outputs.key }} restore-keys: ${{ steps.vcpkg-cache.outputs.restore-key }} + - name: Install compiler cache + id: sccache-setup + if: runner.os != 'Windows' + continue-on-error: true + uses: Mozilla-Actions/sccache-action@v0.0.11 + with: + version: v0.16.0 + disable_annotations: true + - name: Prepare compiler launcher + id: compiler-cache + env: + COMPILER_CACHE_IDENTITY: ${{ steps.vcpkg-cache.outputs.key }} + SCCACHE_SETUP_OUTCOME: ${{ steps.sccache-setup.outcome }} + run: node .github/scripts/compiler-cache.mjs --prepare + - name: Restore compiler cache + id: compiler-restore + if: steps.compiler-cache.outputs.enabled == 'true' + continue-on-error: true + uses: actions/cache/restore@v4 + with: + path: ${{ steps.compiler-cache.outputs.directory }} + key: ${{ steps.compiler-cache.outputs.archive_key }} + restore-keys: ${{ steps.compiler-cache.outputs.restore_keys }} + - name: Start compiler cache after archive restore + id: compiler-start + if: steps.compiler-cache.outputs.enabled == 'true' + env: + COMPILER_CACHE_IDENTITY: ${{ steps.vcpkg-cache.outputs.key }} + SCCACHE_SETUP_OUTCOME: ${{ steps.sccache-setup.outcome }} + SCCACHE_RESTORE_OUTCOME: ${{ steps.compiler-restore.outcome }} + run: node .github/scripts/compiler-cache.mjs - uses: lukka/get-cmake@latest - uses: lukka/run-vcpkg@v11 with: @@ -200,6 +279,8 @@ jobs: "-DCMAKE_TOOLCHAIN_FILE=$env:VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" "-DVCPKG_MANIFEST_FEATURES=application;boost;etcd-client" -DCMAKE_BUILD_TYPE=Debug + "-DCMAKE_C_COMPILER_LAUNCHER=$env:ARK_COMPILER_LAUNCHER" + "-DCMAKE_CXX_COMPILER_LAUNCHER=$env:ARK_COMPILER_LAUNCHER" ${{ runner.os == 'Windows' && '-DVCPKG_APPLOCAL_DEPS=OFF' || '' }} -DARK_BUILD_APPLICATIONS=ON -DARK_BUILD_BENCHMARKS=ON @@ -229,6 +310,21 @@ jobs: - name: Check entity benchmark CLI without measurements run: cmake --build ci-build --target check-entity-benchmark-smoke + - name: Report native cache statistics + if: ${{ !cancelled() }} + continue-on-error: true + env: + VCPKG_ARCHIVE_DIRECTORY: ${{ steps.vcpkg-cache.outputs.directory }} + VCPKG_ARCHIVE_HIT: ${{ steps.vcpkg-restore.outputs.cache-hit }} + COMPILER_ARCHIVE_HIT: ${{ steps.compiler-restore.outputs.cache-hit }} + run: node .github/scripts/cache-stats.mjs + - name: Save compiler cache + if: ${{ success() && steps.compiler-start.outputs.enabled == 'true' && steps.compiler-restore.outputs.cache-hit != 'true' }} + continue-on-error: true + uses: actions/cache/save@v4 + with: + path: ${{ steps.compiler-cache.outputs.directory }} + key: ${{ steps.compiler-cache.outputs.archive_key }} database: name: Database - ${{ matrix.os }} - ${{ matrix.asio }} needs: scope @@ -268,6 +364,37 @@ jobs: path: ${{ steps.vcpkg-cache.outputs.directory }} key: ${{ steps.vcpkg-cache.outputs.key }} restore-keys: ${{ steps.vcpkg-cache.outputs.restore-key }} + - name: Install compiler cache + id: sccache-setup + if: runner.os != 'Windows' + continue-on-error: true + uses: Mozilla-Actions/sccache-action@v0.0.11 + with: + version: v0.16.0 + disable_annotations: true + - name: Prepare compiler launcher + id: compiler-cache + env: + COMPILER_CACHE_IDENTITY: ${{ steps.vcpkg-cache.outputs.key }} + SCCACHE_SETUP_OUTCOME: ${{ steps.sccache-setup.outcome }} + run: node .github/scripts/compiler-cache.mjs --prepare + - name: Restore compiler cache + id: compiler-restore + if: steps.compiler-cache.outputs.enabled == 'true' + continue-on-error: true + uses: actions/cache/restore@v4 + with: + path: ${{ steps.compiler-cache.outputs.directory }} + key: ${{ steps.compiler-cache.outputs.archive_key }} + restore-keys: ${{ steps.compiler-cache.outputs.restore_keys }} + - name: Start compiler cache after archive restore + id: compiler-start + if: steps.compiler-cache.outputs.enabled == 'true' + env: + COMPILER_CACHE_IDENTITY: ${{ steps.vcpkg-cache.outputs.key }} + SCCACHE_SETUP_OUTCOME: ${{ steps.sccache-setup.outcome }} + SCCACHE_RESTORE_OUTCOME: ${{ steps.compiler-restore.outcome }} + run: node .github/scripts/compiler-cache.mjs - uses: lukka/get-cmake@latest - uses: lukka/run-vcpkg@v11 with: @@ -281,6 +408,8 @@ jobs: -DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON "-DVCPKG_MANIFEST_FEATURES=${{ matrix.features }}" -DCMAKE_BUILD_TYPE=Debug + "-DCMAKE_C_COMPILER_LAUNCHER=$env:ARK_COMPILER_LAUNCHER" + "-DCMAKE_CXX_COMPILER_LAUNCHER=$env:ARK_COMPILER_LAUNCHER" -DARK_BUILD_APPLICATIONS=OFF -DARK_BUILD_PROTO=OFF -DARK_BUILD_TESTS=OFF @@ -304,6 +433,21 @@ jobs: run: cmake --build ci-database --target check-database - name: Run local database benchmark self-tests run: cmake --build ci-database --target check-database-benchmark + - name: Report native cache statistics + if: ${{ !cancelled() }} + continue-on-error: true + env: + VCPKG_ARCHIVE_DIRECTORY: ${{ steps.vcpkg-cache.outputs.directory }} + VCPKG_ARCHIVE_HIT: ${{ steps.vcpkg-restore.outputs.cache-hit }} + COMPILER_ARCHIVE_HIT: ${{ steps.compiler-restore.outputs.cache-hit }} + run: node .github/scripts/cache-stats.mjs + - name: Save compiler cache + if: ${{ success() && steps.compiler-start.outputs.enabled == 'true' && steps.compiler-restore.outputs.cache-hit != 'true' }} + continue-on-error: true + uses: actions/cache/save@v4 + with: + path: ${{ steps.compiler-cache.outputs.directory }} + key: ${{ steps.compiler-cache.outputs.archive_key }} required: name: ${{ github.event_name == 'pull_request' && toJSON(github.event.pull_request.draft) == 'true' && 'CI - draft preview' || 'CI - required' }} if: always() diff --git a/AGENTS.md b/AGENTS.md index 791cc8bcf..378a57c14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,10 @@ branch or worktree when the existing checkout contains uncommitted changes. - The formatter selects Git-tracked files. Stage intended new C++ files before running it, or format those paths explicitly. Run the complete check again after committing so newly tracked tests and headers cannot escape validation. +- Keep optional compiler caching separate from vcpkg archives. Preserve compiler + flags, matrix coverage and failure propagation; use an empty launcher when cache + setup is unavailable. Windows Debug remains uncached without changing `/Zi`. + Record hosted cold/warm cache evidence before claiming a speed improvement. - CI policy tests: `node --test tests/ci/*.test.mjs`. Nimbus documentation: `npm ci --prefix docs/site` followed by `npm run check --prefix docs/site`. - Keep official plugins in the ark repository and application business plugins @@ -209,7 +213,7 @@ branch or worktree when the existing checkout contains uncommitted changes. succeeded for the intended source revision and inspect affected published pages. A local preview, merged PR and deployed site are distinct evidence states. - Use the full names AddressSanitizer, UndefinedBehaviorSanitizer and - ThreadSanitizer in explanatory prose. Preserve command flags, paths, workflow + ThreadSanitizer in explanatory prose and human-readable job/step display names. Preserve command flags, paths, workflow identifiers and machine-readable fields. Call Asio/Beast dependencies rather than backends; capitalize Standalone Asio in prose. - Prefer SVG or Mermaid for editable diagrams and charts. Show labeled units and diff --git a/docs/project/contributing.md b/docs/project/contributing.md index 5ccb07e0e..157c15b20 100644 --- a/docs/project/contributing.md +++ b/docs/project/contributing.md @@ -48,7 +48,7 @@ runs that cannot access each other's entries. Missing identity, unsupported dependency layout or archive paths keep persistence disabled with `VCPKG_BINARY_SOURCES=clear`. Cache-service failures retain ordinary source configuration, compilation and regressions. The cache covers archives only, -not installed trees or project object files. Local Node tests validate the +not installed trees. Project object caching is configured separately below. Local Node tests validate the lookup boundaries, exports and workflow wiring; hosted cache reuse and any timing improvement require separate CI evidence. @@ -82,3 +82,52 @@ runs to narrow an ABI difference; equal reported fields do not establish equal unreported inputs. This report is neither a cache-hit verdict nor a performance measurement. In the investigated Windows pair, all 74 package ABIs differed; the particular input causing that difference remains unproven. + + +## CI compiler object cache + +Linux and macOS native jobs optionally use `Mozilla-Actions/sccache-action@v0.0.11` +with sccache `v0.16.0`. Preparation verifies the exact executable version before +exporting both C and C++ CMake compiler launchers. The compiler remains selected +by CMake; all platforms, dependency profiles, sanitizer flags, build targets, +parallelism, regressions and timeouts retain their existing settings. Windows +Debug jobs compile directly: their existing `/Zi` shared PDB configuration is not +changed to force compiler-cache hits. + +The compiler archive has its own namespace, separate from the vcpkg archive. +Its compatibility prefix uses the complete existing dependency key, including +its exact image version, manifest/overlay fingerprint, compiler environment and +module/sanitizer/dependency profile. The source commit is only an immutable +snapshot suffix. There is no run ID or source commit in the compatibility prefix +or `SCCACHE_C_CUSTOM_CACHE_BUSTER`, so another commit with the same identity can +restore objects. There is no cross-image or cross-profile compiler restore. +sccache still validates compiler identity, compile arguments and source/header +inputs for every invocation; an archive restore alone does not establish an +object hit or bypass compilation validation. + +Each job sets `SCCACHE_CACHE_SIZE=512M` under the runner temporary directory. +The full matrix has nine eligible Linux/macOS jobs, giving nine such local cache +budgets per source snapshot; Windows adds no compiler cache. Actions stores +immutable snapshots, which may overlap across commits and refs, so this is not a +repository-wide storage cap. Compression, metadata, dependency archives and cache +service retention are separate. Actions branch/ref visibility still applies; +queue refs cannot read each other's entries merely because their keys match. + +Installation, version validation, cache restoration or server startup failures +leave an empty launcher and normal compilation. Cache service I/O failures use +`SCCACHE_IGNORE_SERVER_IO_ERROR=1`. Compiler and test failures still fail the job; +the workflow never retries a compiler failure without the launcher. Only successful +jobs save new compiler snapshots. A best-effort job summary reports dependency +archive availability, compiler archive exact hits and sccache statistics. Prefix +restoration and object hits are different observations. + +Run `node --test tests/ci/*.test.mjs` for the cache identity/fallback and workflow +contracts. The permanent CMake/Ninja fixture compiles C and C++ through a launcher +path containing spaces and checks that intentional compiler errors propagate +with and without a launcher. It uses a transparent fixture launcher, not sccache, +and does not prove hosted cache hits. Hosted acceptance must compare a cold job +and a subsequent job with the same identity, record the source revisions, +archive lookup, sccache hits/misses and setup/configure/build/test durations, +and retain all required regressions. No build-time improvement is claimed before +that evidence is available. Native CI also needs separate Linux/macOS acceptance; +Windows remains uncached. diff --git a/docs/project/contributing_CN.md b/docs/project/contributing_CN.md index b509aea20..611db3369 100644 --- a/docs/project/contributing_CN.md +++ b/docs/project/contributing_CN.md @@ -39,7 +39,7 @@ Actions 缓存的分支／引用可见性规则继续生效，详见 [Actions 相互无法读取缓存的队列运行需要默认分支中兼容的 v2 归档作为初始来源。 身份信息缺失、依赖布局或归档路径不受支持时，继续关闭持久缓存并导出 `VCPKG_BINARY_SOURCES=clear`。 -缓存服务失败不影响正常的源码配置、编译和回归检查。缓存范围仅为包归档，不包含依赖安装目录或项目目标文件。 +缓存服务失败不影响正常的源码配置、编译和回归检查。依赖缓存仅覆盖包归档，不包含依赖安装目录。项目目标文件缓存单独配置，见下文。 本地 Node 测试验证查找边界、输出与工作流连接；托管环境中的缓存复用和耗时变化仍需独立 CI 证据。 @@ -63,3 +63,38 @@ Actions 缓存的分支／引用可见性规则继续生效，详见 [Actions 对比不同运行的指纹可以缩小 ABI 差异范围；已输出字段相同不能证明未输出的输入相同。 诊断不判定缓存命中，也不构成性能测量。此前分析的两轮 Windows 日志中，74 个包的 ABI 均发生变化； 导致变化的具体输入仍未得到证实。 + + +## CI 编译目标文件缓存 + +Linux 和 macOS 原生任务可选使用 `Mozilla-Actions/sccache-action@v0.0.11`， +固定 sccache 版本为 `v0.16.0`。准备步骤先验证可执行文件的精确版本，再导出 C 和 C++ 的 +CMake 编译器 launcher。编译器仍由 CMake 选择；平台、依赖 profile、sanitizer 参数、构建目标、 +并行度、回归检查和超时设置保持不变。Windows Debug 任务直接编译；保留现有 `/Zi` 共享 PDB +配置，不为获得缓存命中而修改参数。 + +编译归档使用独立命名空间，与 vcpkg 包归档分开。兼容前缀采用现有依赖 key 的完整身份信息， +包括精确镜像版本、manifest/overlay 指纹、编译器环境，以及模块、sanitizer 和依赖 profile。 +源码 commit 只作为不可变快照的后缀。兼容前缀和 `SCCACHE_C_CUSTOM_CACHE_BUSTER` 不包含运行 ID +或源码 commit，因此相同身份的后续 commit 可以恢复目标文件。编译缓存不跨镜像或 profile 回退。 +sccache 仍逐次验证编译器身份、编译参数及源码和头文件输入；恢复归档不代表目标文件命中， +也不会绕过编译输入校验。 + +每个任务在执行机临时目录中设置 `SCCACHE_CACHE_SIZE=512M`。完整矩阵有九个可用的 Linux/macOS +任务，每个源码快照对应九份这样的本地缓存预算；Windows 不增加编译缓存。 +Actions 保存不可变快照，不同 commit 和引用的快照可能重复占用空间，因此这不是仓库总存储上限。 +压缩、元数据、依赖包归档和缓存服务保留策略需要单独计算。Actions 的分支／引用可见性规则仍然适用； +key 相同不能让不同合并队列引用相互读取缓存。 + +安装、版本验证、缓存恢复或服务启动失败时，launcher 为空并正常编译。 +缓存服务 I/O 失败通过 `SCCACHE_IGNORE_SERVER_IO_ERROR=1` 回退。 +编译或测试失败仍使任务失败；工作流不会在编译器报错后移除 launcher 自动重试。 +只有成功的任务保存新的编译快照。尽力生成的任务摘要报告依赖归档状态、编译归档精确命中状态， +以及 sccache 统计。前缀恢复与目标文件命中是两类不同证据。 + +运行 `node --test tests/ci/*.test.mjs` 验证缓存身份、回退行为和工作流契约。 +永久保留的 CMake/Ninja fixture 使用含空格路径的 launcher 编译 C 和 C++，并验证启用和关闭 +launcher 时，故意引入的编译错误均会传播。fixture 使用透明转发脚本，不使用 sccache， +不能证明托管环境命中。托管验收需要对比冷任务与随后身份相同的任务，记录源码版本、归档查找结果、 +sccache 命中／未命中，以及准备、配置、构建和测试耗时，并保留全部必要回归。 +这些证据齐备之前不宣称构建提速。Linux/macOS 原生 CI 仍需分别验收；Windows 继续直接编译。 diff --git a/tests/ci/cache-workflow.test.mjs b/tests/ci/cache-workflow.test.mjs index 487a06623..f7a44aa75 100644 --- a/tests/ci/cache-workflow.test.mjs +++ b/tests/ci/cache-workflow.test.mjs @@ -31,7 +31,7 @@ for (const [name, configureName, buildDirectory, checks] of profiles) { test(`${name} saves dependency archives immediately after successful configure`, () => { const steps = jobSteps(name); const configure = onlyStep(steps, step => step.startsWith(` - name: ${configureName}\n`), 'configure step'); - const save = onlyStep(steps, step => step.includes('uses: actions/cache/save@v4\n'), 'archive save'); + const save = onlyStep(steps, step => step.includes('name: Save vcpkg package archives\n'), 'archive save'); const build = onlyStep(steps, step => step.includes(`run: cmake --build ${buildDirectory} --parallel `), 'source build'); assert.equal(save.index, configure.index + 1, 'save only after configure has completed'); assert.ok(save.index < build.index, 'save dependencies before compiling project sources'); @@ -44,8 +44,8 @@ for (const [name, configureName, buildDirectory, checks] of profiles) { test(`${name} restores exact archives with one compatible prefix and saves only the exact key`, () => { const steps = jobSteps(name); const prepare = onlyStep(steps, step => step.includes('id: vcpkg-cache\n'), 'cache identity'); - const restore = onlyStep(steps, step => step.includes('uses: actions/cache/restore@v4\n'), 'archive restore'); - const save = onlyStep(steps, step => step.includes('uses: actions/cache/save@v4\n'), 'archive save'); + const restore = onlyStep(steps, step => step.includes('id: vcpkg-restore\n'), 'archive restore'); + const save = onlyStep(steps, step => step.includes('name: Save vcpkg package archives\n'), 'archive save'); const toolchain = onlyStep(steps, step => step.includes('uses: lukka/run-vcpkg@v11\n'), 'vcpkg action'); assert.ok(prepare.index < restore.index && restore.index < toolchain.index); assert.match(restore.step, /^ if: steps\.vcpkg-cache\.outputs\.enabled == 'true'$/m); @@ -55,7 +55,9 @@ for (const [name, configureName, buildDirectory, checks] of profiles) { assert.match(restore.step, /^ restore-keys: \$\{\{ steps\.vcpkg-cache\.outputs\.restore-key \}\}$/m); assert.equal((restore.step.match(/restore-keys:/g) ?? []).length, 1); assert.doesNotMatch(save.step, /restore-keys:/); - assert.equal(steps.filter(step => step.includes('uses: actions/cache')).length, 2); + assert.equal(steps.filter(step => step.includes('uses: actions/cache')).length, 4); + assert.match(save.step, /uses: actions\/cache\/save@v4/); + assert.match(restore.step, /uses: actions\/cache\/restore@v4/); for (const { step } of [prepare, toolchain]) { assert.doesNotMatch(step, /VCPKG_DISABLE_COMPILER_TRACKING|VCPKG_ENV_PASSTHROUGH_UNTRACKED|--editable/); } @@ -77,3 +79,38 @@ for (const [name, configureName, buildDirectory, checks] of profiles) { } }); } + +for (const [name, configureName] of profiles) { + test(`${name} restores optional pinned compiler objects before starting the launcher`, () => { + const steps = jobSteps(name); + const find = id => onlyStep(steps, step => step.includes(`id: ${id}\n`), id); + const setup = find('sccache-setup'); + const prepare = find('compiler-cache'); + const restore = find('compiler-restore'); + const start = find('compiler-start'); + const configure = onlyStep(steps, step => step.startsWith(` - name: ${configureName}\n`), 'configure step'); + const save = onlyStep(steps, step => step.includes('name: Save compiler cache\n'), 'compiler save'); + assert.ok(setup.index < prepare.index && prepare.index < restore.index && restore.index < start.index && start.index < configure.index); + assert.match(setup.step, /uses: Mozilla-Actions\/sccache-action@v0\.0\.11/); + assert.match(setup.step, /version: v0\.16\.0/); + assert.match(setup.step, /disable_annotations: true/); + assert.match(setup.step, /if: runner\.os != 'Windows'/); + for (const { step } of [setup, restore, save]) assert.match(step, /continue-on-error: true/); + for (const { step } of [prepare, start]) { + assert.match(step, /COMPILER_CACHE_IDENTITY: \$\{\{ steps\.vcpkg-cache\.outputs\.key \}\}/); + assert.match(step, /SCCACHE_SETUP_OUTCOME: \$\{\{ steps\.sccache-setup\.outcome \}\}/); + } + assert.match(prepare.step, /compiler-cache\.mjs --prepare/); + assert.match(start.step, /SCCACHE_RESTORE_OUTCOME: \$\{\{ steps\.compiler-restore\.outcome \}\}/); + assert.match(restore.step, /key: \$\{\{ steps\.compiler-cache\.outputs\.archive_key \}\}/); + assert.match(restore.step, /restore-keys: \$\{\{ steps\.compiler-cache\.outputs\.restore_keys \}\}/); + for (const language of ['C', 'CXX']) { + assert.ok(configure.step.includes(`"-DCMAKE_${language}_COMPILER_LAUNCHER=$env:ARK_COMPILER_LAUNCHER"`)); + } + assert.match(save.step, /if: \$\{\{ success\(\) && steps\.compiler-start\.outputs\.enabled == 'true' && steps\.compiler-restore\.outputs\.cache-hit != 'true' \}\}/); + assert.match(save.step, /path: \$\{\{ steps\.compiler-cache\.outputs\.directory \}\}/); + assert.match(save.step, /key: \$\{\{ steps\.compiler-cache\.outputs\.archive_key \}\}/); + assert.equal(save.index, steps.length - 1, 'save only after all source builds and checks'); + assert.doesNotMatch(configure.step, /CMAKE_(?:C|CXX)_FLAGS=.*\/(?:Zi|Z7)/); + }); +} diff --git a/tests/ci/compiler-cache.test.mjs b/tests/ci/compiler-cache.test.mjs new file mode 100644 index 000000000..b0ea500dd --- /dev/null +++ b/tests/ci/compiler-cache.test.mjs @@ -0,0 +1,170 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { cachePlan } from '../../.github/scripts/vcpkg-cache.mjs'; +import { prepareCompilerCache, sccacheVersion } from '../../.github/scripts/compiler-cache.mjs'; + +function fixture(t) { + const root = mkdtempSync(join(tmpdir(), 'ark-compiler-cache-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + mkdirSync(join(root, 'ports')); + writeFileSync(join(root, 'vcpkg.json'), '{}'); + writeFileSync(join(root, 'vcpkg-configuration.json'), '{"overlay-ports":["ports"]}'); + const env = { + GITHUB_SHA: 'b'.repeat(40), GITHUB_RUN_ID: '1001', GITHUB_RUN_ATTEMPT: '1', + RUNNER_OS: 'Linux', RUNNER_ARCH: 'X64', RUNNER_TEMP: join(root, 'runner temp'), + ImageOS: 'ubuntu24', ImageVersion: '20261020.1', VCPKG_CACHE_REVISION: 'a'.repeat(40), + VCPKG_CACHE_FEATURES: 'boost;ssl', VCPKG_CACHE_DEFAULT_FEATURES: 'true', + VCPKG_CACHE_PROFILE: 'ON-ON-OFF-Release', VCPKG_DEFAULT_TRIPLET: 'x64-linux', + VCPKG_DEFAULT_HOST_TRIPLET: 'x64-linux', GITHUB_ENV: join(root, 'env'), + GITHUB_OUTPUT: join(root, 'output'), SCCACHE_SETUP_OUTCOME: 'success', + SCCACHE_PATH: '/tool cache/sccache', SCCACHE_RESTORE_OUTCOME: 'success', + }; + env.COMPILER_CACHE_IDENTITY = cachePlan({ root, env }).key; + return { root, env }; +} + +test('compiler launcher uses only the verified pinned version with cache IO fallback', t => { + const f = fixture(t); + const calls = []; + const plan = prepareCompilerCache({ ...f, run: (command, args, options) => { + calls.push([command, args]); + assert.equal(options.env.SCCACHE_GHA_ENABLED, 'false'); + assert.equal(options.env.SCCACHE_IGNORE_SERVER_IO_ERROR, '1'); + assert.match(options.env.SCCACHE_C_CUSTOM_CACHE_BUSTER, /ark-sccache-0.16.0-v1/); + return { status: 0, stdout: `sccache ${sccacheVersion}\n` }; + } }); + assert.equal(plan.enabled, true); + assert.equal(plan.launcher, f.env.SCCACHE_PATH); + assert.deepEqual(calls.map(call => call[1]), [['--version'], ['--start-server']]); + assert.match(readFileSync(f.env.GITHUB_ENV, 'utf8'), /ARK_COMPILER_LAUNCHER=\/tool cache\/sccache/); +}); + +test('preparation does not start a server before Actions restores the local cache directory', t => { + const f = fixture(t); + const calls = []; + const plan = prepareCompilerCache({ ...f, start: false, run: (_command, args) => { + calls.push(args); + return { status: 0, stdout: `sccache ${sccacheVersion}\n` }; + } }); + assert.equal(plan.enabled, true); + assert.deepEqual(calls, [['--version']]); +}); + +test('failed setup, wrong version, startup errors and incomplete identity all use the compiler directly', t => { + for (const extra of [{ SCCACHE_SETUP_OUTCOME: 'failure' }, { SCCACHE_RESTORE_OUTCOME: 'failure' }, { SCCACHE_PATH: '' }, + { SCCACHE_PATH: '/unsafe;launcher' }, { SCCACHE_PATH: 'relative/launcher' }, { RUNNER_ARCH: 'ARM64' }, { COMPILER_CACHE_IDENTITY: '' }, { COMPILER_CACHE_IDENTITY: 'unsafe\nidentity' }, + { GITHUB_SHA: '' }, { RUNNER_OS: 'Windows' }, { RUNNER_TEMP: '/unsafe;cache' }, + { behavior: 'wrong-version' }, { behavior: 'version-error' }, { behavior: 'startup-error' }, { behavior: 'throw' }]) { + const f = fixture(t); + const calls = []; + const plan = prepareCompilerCache({ env: { ...f.env, ...extra }, run: (_command, args) => { + calls.push(args); + if (extra.behavior === 'throw') throw new Error('tool unavailable'); + return { status: extra.behavior === 'version-error' || (extra.behavior === 'startup-error' && args[0] === '--start-server') ? 1 : 0, + stdout: extra.behavior === 'wrong-version' ? 'sccache 0.1.0' : `sccache ${sccacheVersion}` }; + } }); + assert.equal(plan.enabled, false, JSON.stringify(extra)); + if (!extra.behavior) assert.deepEqual(calls, [], JSON.stringify(extra)); + assert.equal(plan.launcher, ''); + assert.match(readFileSync(f.env.GITHUB_ENV, 'utf8'), /^ARK_COMPILER_LAUNCHER=\n/); + } +}); + +test('CMake passes a launcher path with spaces, keeps the compiler and propagates compilation failures', { skip: process.platform === 'win32' }, t => { + const f = fixture(t); + const launcher = join(f.root, 'launcher with spaces.mjs'); + const trace = join(f.root, 'launcher-trace'); + // Run a transparent fixture launcher through the same two CMake launcher arguments as CI. + writeFileSync(launcher, `import { spawnSync } from 'node:child_process';\nimport { appendFileSync } from 'node:fs';\nappendFileSync(${JSON.stringify(trace)}, process.argv[2] + '\\n');\nconst r = spawnSync(process.argv[2], process.argv.slice(3), { stdio: 'inherit' });\nprocess.exit(r.status ?? 1);\n`); + writeFileSync(join(f.root, 'CMakeLists.txt'), 'cmake_minimum_required(VERSION 3.21)\nproject(launcher LANGUAGES C CXX)\nadd_executable(check main.cpp helper.c)\n'); + writeFileSync(join(f.root, 'main.cpp'), 'int main() { return 0; }\n'); + writeFileSync(join(f.root, 'helper.c'), 'int helper(void) { return 0; }\n'); + const build = join(f.root, 'build'); + const run = args => spawnSync('cmake', args, { encoding: 'utf8', timeout: 30_000 }); + const configured = run(['-S', f.root, '-B', build, '-G', 'Ninja', + `-DCMAKE_C_COMPILER_LAUNCHER=${process.execPath};${launcher}`, + `-DCMAKE_CXX_COMPILER_LAUNCHER=${process.execPath};${launcher}`]); + assert.equal(configured.status, 0, configured.stdout + configured.stderr); + const compiled = run(['--build', build, '--parallel', '1']); + assert.equal(compiled.status, 0, compiled.stdout + compiled.stderr); + const compilers = readFileSync(trace, 'utf8').trim().split('\n'); + assert.equal(compilers.length, 2); + const cache = readFileSync(join(build, 'CMakeCache.txt'), 'utf8'); + for (const language of ['C', 'CXX']) { + const compiler = cache.match(new RegExp(`^CMAKE_${language}_COMPILER:FILEPATH=(.+)$`, 'm'))?.[1]; + assert.ok(compiler && compilers.includes(compiler), `${language}: CMake retains the actual compiler`); + } + writeFileSync(join(f.root, 'main.cpp'), '#error intentional compiler failure\n'); + const failed = run(['--build', build, '--parallel', '1']); + assert.notEqual(failed.status, 0); + assert.match(failed.stdout + failed.stderr, /intentional compiler failure/); + writeFileSync(join(f.root, 'main.cpp'), 'int main() { return 0; }\n'); + const direct = join(f.root, 'direct'); + assert.equal(run(['-S', f.root, '-B', direct, '-G', 'Ninja', + '-DCMAKE_C_COMPILER_LAUNCHER=', '-DCMAKE_CXX_COMPILER_LAUNCHER=']).status, 0); + assert.equal(run(['--build', direct, '--parallel', '1']).status, 0, 'disabled caches compile normally'); + writeFileSync(join(f.root, 'main.cpp'), '#error intentional compiler failure\n'); + assert.notEqual(run(['--build', direct, '--parallel', '1']).status, 0, 'cold builds must preserve compiler errors'); +}); + +test('source commits and run attempts share object compatibility while immutable snapshots differ', t => { + const f = fixture(t); + const run = () => ({ status: 0, stdout: `sccache ${sccacheVersion}` }); + const first = prepareCompilerCache({ ...f, run }); + const next = prepareCompilerCache({ ...f, env: { ...f.env, GITHUB_SHA: 'c'.repeat(40), GITHUB_RUN_ID: '2002', GITHUB_RUN_ATTEMPT: '2' }, run }); + assert.equal(first.key, next.key); + assert.equal(first.restoreKeys, next.restoreKeys); + assert.notEqual(first.archiveKey, next.archiveKey); + assert.ok(first.archiveKey.startsWith(next.restoreKeys)); + assert.equal(first.key.includes(f.env.GITHUB_SHA), false); + assert.equal(first.directory, next.directory); +}); + +test('runner image, compiler flags, dependency contents and profiles isolate compiler restore prefixes', t => { + const f = fixture(t); + const run = () => ({ status: 0, stdout: `sccache ${sccacheVersion}` }); + const baseline = prepareCompilerCache({ ...f, run }); + for (const extra of [{ ImageVersion: 'different-image' }, { CXXFLAGS: '-fsanitize=thread' }, + { CXX: '/other/compiler' }, { VCPKG_CACHE_PROFILE: 'database-Standalone' }, + { VCPKG_CACHE_FEATURES: 'database' }, { VCPKG_CACHE_REVISION: 'd'.repeat(40) }]) { + const identity = cachePlan({ root: f.root, env: { ...f.env, ...extra } }).key; + const plan = prepareCompilerCache({ ...f, env: { ...f.env, COMPILER_CACHE_IDENTITY: identity }, run }); + assert.notEqual(plan.restoreKeys, baseline.restoreKeys, JSON.stringify(extra)); + } + writeFileSync(join(f.root, 'vcpkg.json'), '{"dependencies":["zlib"]}'); + const identity = cachePlan(f).key; + assert.notEqual(prepareCompilerCache({ ...f, env: { ...f.env, COMPILER_CACHE_IDENTITY: identity }, run }).restoreKeys, baseline.restoreKeys); +}); + +test('unavailable or symbolic-link local storage stays cold and missing command channels fail explicitly', { skip: process.platform === 'win32' }, t => { + const f = fixture(t); + writeFileSync(f.env.RUNNER_TEMP, 'not a directory'); + const run = () => ({ status: 0, stdout: `sccache ${sccacheVersion}` }); + assert.equal(prepareCompilerCache({ ...f, run }).enabled, false); + rmSync(f.env.RUNNER_TEMP); + mkdirSync(f.env.RUNNER_TEMP); + symlinkSync(f.root, join(f.env.RUNNER_TEMP, 'ark-sccache')); + assert.equal(prepareCompilerCache({ ...f, run }).enabled, false); + for (const name of ['GITHUB_ENV', 'GITHUB_OUTPUT']) { + assert.throws(() => prepareCompilerCache({ ...f, env: { ...f.env, [name]: '' } }), /file-command/); + } +}); + + +test('failed restore or startup clears the previously prepared launcher before CMake runs', t => { + for (const failure of ['restore', 'startup']) { + const f = fixture(t); + const ok = () => ({ status: 0, stdout: `sccache ${sccacheVersion}` }); + assert.equal(prepareCompilerCache({ ...f, start: false, run: ok }).enabled, true); + const env = { ...f.env, SCCACHE_RESTORE_OUTCOME: failure === 'restore' ? 'failure' : 'success' }; + const run = (_command, args) => ({ status: args[0] === '--start-server' ? 1 : 0, stdout: `sccache ${sccacheVersion}` }); + assert.equal(prepareCompilerCache({ env, run }).enabled, false); + const launchers = readFileSync(f.env.GITHUB_ENV, 'utf8').split('\n').filter(line => line.startsWith('ARK_COMPILER_LAUNCHER=')); + assert.deepEqual(launchers, [`ARK_COMPILER_LAUNCHER=${f.env.SCCACHE_PATH}`, 'ARK_COMPILER_LAUNCHER=']); + assert.ok(readFileSync(f.env.GITHUB_OUTPUT, 'utf8').endsWith('enabled=false\ndirectory=\nkey=\narchive_key=\nrestore_keys=\n')); + } +}); From 1c7eb38eaa92c2b7a48e2a960bd5e244d3ad5346 Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 23 Oct 2026 17:35:12 +0800 Subject: [PATCH 5/6] fix(build): prefer configuration directories when finding LZ4 --- cmake/FindLZ4.cmake | 2 +- docs/dependencies.md | 18 +++++++++++ docs/dependencies_CN.md | 13 ++++++++ tests/ci/lz4-finder.test.mjs | 61 ++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 tests/ci/lz4-finder.test.mjs diff --git a/cmake/FindLZ4.cmake b/cmake/FindLZ4.cmake index 4658a35bf..2d4bf32cd 100644 --- a/cmake/FindLZ4.cmake +++ b/cmake/FindLZ4.cmake @@ -7,7 +7,7 @@ # find_path(LZ4_INCLUDE_DIR NAMES lz4.h) -find_library(LZ4_LIBRARY NAMES lz4) +find_library(LZ4_LIBRARY NAMES lz4 lz4d NAMES_PER_DIR) # We require LZ4_compress_default() which was added in v1.7.0 if (LZ4_LIBRARY) diff --git a/docs/dependencies.md b/docs/dependencies.md index fe9ccc2a9..0df69446e 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -41,6 +41,24 @@ arknet uses [0.2.3](https://github.com/OpenArkStudio/arknet/releases/tag/v0.2.3) For local development, supply `ARKNET_SOURCE_DIR`. Existing build directories retain their cached `ARKNET_REVISION`; follow the [installation instructions](get-started/installation.md#upgrade-arknet) to update it explicitly, or use a fresh build directory. Existing performance and database reports continue to describe their stated tested revisions. +## Windows Debug compression dependencies + +The LZ4 finder accepts `lz4` and `lz4d` basenames and searches each configured +directory before moving to the next. This preserves the toolchain's configuration +preference: a Debug `debug/lib/lz4d.lib` takes precedence over a later release +`lib/lz4.lib`; Release still prefers its release directory. Zstandard discovery +and the compiler settings are unchanged. + +If a Windows compression regression exits with `0xc0000135` before doctest starts, +inspect `LZ4_LIBRARY` in `CMakeCache.txt` and the selected import library's DLL +dependencies. A release LZ4 import library selected in Debug can require a DLL +outside the configured `debug/bin` runtime directory. After updating this finder, +clear the cached selection with `cmake -U LZ4_LIBRARY -S . -B ` and repeat +the original configure options, or use a fresh build directory. The CI runtime +helper continues to export only the matching configuration directory; do not mix +release DLLs into the Debug runtime path. Finder fixtures verify search selection +without compiling fake libraries; Windows execution remains a separate check. + ## Optional legacy integrations The physic plugin and physics_scene depend on PhysX 3 and have no validated SDK diff --git a/docs/dependencies_CN.md b/docs/dependencies_CN.md index 33143bfcd..29e48fc38 100644 --- a/docs/dependencies_CN.md +++ b/docs/dependencies_CN.md @@ -36,6 +36,19 @@ arknet 使用 [0.2.3](https://github.com/OpenArkStudio/arknet/releases/tag/v0.2. 本地开发可以通过 `ARKNET_SOURCE_DIR` 提供源码。已有构建目录会保留缓存中的 `ARKNET_REVISION`，升级时请按[安装说明](get-started/installation_CN.md#升级-arknet)显式更新缓存，或使用新的构建目录。既有性能与数据库报告仍对应各自列出的测试版本。 +## Windows Debug 压缩依赖 + +LZ4 finder 支持 `lz4` 和 `lz4d` 名称，先检查每个已配置目录中的候选名称，再移到下一个目录。 +这保留工具链的配置优先级：Debug 的 `debug/lib/lz4d.lib` 优先于后续 Release 目录中的 +`lib/lz4.lib`；Release 仍优先查找自己的目录。Zstandard 查找方式和编译器设置保持不变。 + +Windows 压缩回归如果在 doctest 启动前以 `0xc0000135` 退出，应检查 `CMakeCache.txt` 中的 +`LZ4_LIBRARY`，以及所选导入库依赖的 DLL。Debug 如果选中 Release LZ4 导入库，可能需要 +配置的 `debug/bin` 运行目录之外的 DLL。更新 finder 后，使用 +`cmake -U LZ4_LIBRARY -S . -B ` 清除缓存选择，并重复原配置参数，或使用新的构建目录。 +CI 运行时辅助脚本仍只导出匹配配置的目录，不向 Debug 运行路径混入 Release DLL。 +finder fixture 只验证查找选择，不编译模拟库；Windows 实际运行仍需单独验收。 + ## 可选的旧版集成 physic 插件与 physics_scene 依赖 PhysX 3，当前构建没有已验证的 SDK 集成。 diff --git a/tests/ci/lz4-finder.test.mjs b/tests/ci/lz4-finder.test.mjs new file mode 100644 index 000000000..a09b6bb7f --- /dev/null +++ b/tests/ci/lz4-finder.test.mjs @@ -0,0 +1,61 @@ +// Copyright (c) 2026 OpenArkStudio. SPDX-License-Identifier: Apache-2.0 +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; + +const finder = fileURLToPath(new URL('../../cmake/FindLZ4.cmake', import.meta.url)).replaceAll('\\', '/'); + +function selectLibrary(t, { configuration, preferredName, fallbackName, preferredDirectory = 'debug' }) { + const root = mkdtempSync(join(tmpdir(), 'ark-lz4-finder-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const preferred = join(root, preferredDirectory); + const fallback = join(root, 'fallback'); + for (const [prefix, name] of [[preferred, preferredName], [fallback, fallbackName]]) { + mkdirSync(join(prefix, 'lib'), { recursive: true }); + mkdirSync(join(prefix, 'include')); + writeFileSync(join(prefix, 'lib', name), 'fake import library; must not be linked\n'); + writeFileSync(join(prefix, 'include', 'lz4.h'), '/* finder-only fixture */\n'); + } + writeFileSync(join(root, 'CMakeLists.txt'), `cmake_minimum_required(VERSION 3.21) +project(lz4_finder NONE) +# Fake Windows import libraries: no compiler or ABI/runtime validation. +set(CMAKE_FIND_LIBRARY_PREFIXES "") +set(CMAKE_FIND_LIBRARY_SUFFIXES ".lib") +set(CMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH FALSE) +set(CMAKE_FIND_USE_CMAKE_SYSTEM_PATH FALSE) +set(CMAKE_FIND_USE_INSTALL_PREFIX FALSE) +# The real version probe compiles C; fake files test only search selection. +set(LZ4_GOOD_VERSION TRUE CACHE BOOL "Skip fake-library compilation") +include("${finder}") +if(NOT LZ4_FOUND) + message(FATAL_ERROR "Fixture library was not found") +endif() +file(WRITE "\${CMAKE_BINARY_DIR}/selected.txt" "\${LZ4_LIBRARY}") +`); + const result = spawnSync('cmake', ['-S', root, '-B', join(root, 'build'), + '-DCMAKE_SYSTEM_NAME=Windows', `-DCMAKE_BUILD_TYPE=${configuration}`, + `-DCMAKE_PREFIX_PATH=${preferred};${fallback}`], { encoding: 'utf8', timeout: 30_000 }); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.equal(readFileSync(join(root, 'build', 'selected.txt'), 'utf8'), + join(preferred, 'lib', preferredName).replaceAll('\\', '/')); +} + +test('Debug selects suffixed lz4d before a later release lz4 import library', t => { + selectLibrary(t, { configuration: 'Debug', preferredName: 'lz4d.lib', fallbackName: 'lz4.lib' }); +}); + +test('Debug retains configuration precedence when both libraries have the lz4 basename', t => { + selectLibrary(t, { configuration: 'Debug', preferredName: 'lz4.lib', fallbackName: 'lz4.lib' }); +}); + +test('Release selects its lz4 import library before a later debug lz4d library', t => { + selectLibrary(t, { configuration: 'Release', preferredDirectory: 'release', preferredName: 'lz4.lib', fallbackName: 'lz4d.lib' }); +}); + +test('configured directory precedence outranks alternative library basename order', t => { + selectLibrary(t, { configuration: 'Release', preferredDirectory: 'preferred prefix', preferredName: 'lz4d.lib', fallbackName: 'lz4.lib' }); +}); From c6fa8fb45448e22ae8e0182b3b77c7413b1770eb Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 23 Oct 2026 19:17:09 +0800 Subject: [PATCH 6/6] fix(ci): treat root agent instructions as documentation --- .github/scripts/ci-policy.mjs | 4 +- docs/project/contributing.md | 13 ++++ docs/project/contributing_CN.md | 10 ++++ tests/ci/README.md | 8 +++ tests/ci/README_CN.md | 6 ++ tests/ci/module-validation.test.mjs | 5 +- tests/ci/policy.test.mjs | 93 +++++++++++++++++++++++++++++ 7 files changed, 135 insertions(+), 4 deletions(-) diff --git a/.github/scripts/ci-policy.mjs b/.github/scripts/ci-policy.mjs index b7ba92331..e4a53e88e 100644 --- a/.github/scripts/ci-policy.mjs +++ b/.github/scripts/ci-policy.mjs @@ -22,7 +22,7 @@ export function isDocumentationPath(file) { if (cppPath.test(file) || /^CMake(?:Lists\.txt|(?:User)?Presets\.json|Cache\.txt)$/i.test(name) || /\.cmake$/i.test(file) || dependencyName.test(name) || file.split('/').includes('.github') || /^action\.ya?ml$/i.test(name) || /^\.clang-(?:format|tidy)$/.test(name)) return false; - if (file === 'README.md' || file === 'README_CN.md' || pluginIndexPaths.has(file) || testGuidePaths.has(file)) return true; + if (file === 'AGENTS.md' || file === 'README.md' || file === 'README_CN.md' || pluginIndexPaths.has(file) || testGuidePaths.has(file)) return true; return file.startsWith('docs/') && (docPath.test(file) || name === 'LICENSE' || name === '.gitignore'); } @@ -98,7 +98,7 @@ export function selectScope({ eventName, files, site, enumerationFailed = false const scopedEvent = eventName === 'pull_request' || eventName === 'merge_group'; const docsOnly = scopedEvent && nonempty && site.available && paths.every(isDocumentationPath); const readme = paths.some(file => file === 'README.md' || file === 'README_CN.md'); - const docsTouched = readme || paths.some(file => file.startsWith('docs/') || pluginIndexPaths.has(file) || testGuidePaths.has(file)); + const docsTouched = readme || paths.some(file => file === 'AGENTS.md' || file.startsWith('docs/') || pluginIndexPaths.has(file) || testGuidePaths.has(file)); const ciTouched = paths.some(file => file.startsWith('.github/') || file.startsWith('tests/ci/')); const siteReferenced = site.present || paths.some(file => file.startsWith('docs/site/')); const docs = siteReferenced && (eventName !== 'pull_request' || !known || docsTouched || ciTouched); diff --git a/docs/project/contributing.md b/docs/project/contributing.md index 157c15b20..297512f87 100644 --- a/docs/project/contributing.md +++ b/docs/project/contributing.md @@ -15,6 +15,19 @@ npm run check --prefix docs/site The site provides language switching, search, code highlighting and plain Markdown endpoints for AI readers. Pull requests run the documentation checks without publishing. After a change is merged into `main`, the documentation workflow builds and publishes GitHub Pages; maintainers can also trigger it manually. Node.js 22.12+ is needed for the documentation build; it is not needed to compile ark or its C++ tests. +## CI documentation scope + +The exact root `AGENTS.md` file is documentation. When the changed range is known, +nonempty and confined to recognized documentation, and the Nimbus site is available, +both source PRs and merge-group candidates require the documentation checks and +`CI - required`, while native jobs are explicitly skipped. Nimbus checks the site; +root README link checks run only when a root README changes. + +Nested unknown `AGENTS.md` paths, similarly named code, mixed source changes, +unavailable ranges or an unavailable site retain native checks. Manual runs keep +the full native matrix. Changes to the CI policy itself still require native +acceptance; this documentation exception does not change that requirement. + ## CI dependency archive cache CI restores vcpkg package archives from the runner temporary directory with diff --git a/docs/project/contributing_CN.md b/docs/project/contributing_CN.md index 611db3369..efd4752f8 100644 --- a/docs/project/contributing_CN.md +++ b/docs/project/contributing_CN.md @@ -15,6 +15,16 @@ npm run check --prefix docs/site 站点提供语言切换、搜索、代码高亮，以及供 AI 阅读的纯 Markdown 接口。PR 只运行文档检查，不发布站点。变更合并到 `main` 后，文档工作流构建并发布 GitHub Pages；维护者也可手动触发。文档构建需要 Node.js 22.12+；编译 ark 与 C++ 测试不需要 Node.js。 +## CI 文档范围 + +根目录的精确文件 `AGENTS.md` 归为文档。改动范围已知且非空、仅包含已识别的文档， +并且 Nimbus 文档站可用时，源分支 PR 与合并候选均要求文档检查和 `CI - required`， +原生任务明确跳过。Nimbus 检查文档站；只有根 README 变化时才检查其链接。 + +嵌套的未知 `AGENTS.md` 路径、名称相近的代码、混合源码变更、范围不可用或文档站不可用时， +仍保留原生检查。手动运行保留完整原生矩阵。CI 策略本身的变更仍须完成原生验收， +这项文档例外不会改变该要求。 + ## CI 依赖包归档缓存 CI 使用 `actions/cache` 恢复执行机临时目录中的 vcpkg 包归档。 diff --git a/tests/ci/README.md b/tests/ci/README.md index fa7d1423f..60a681737 100644 --- a/tests/ci/README.md +++ b/tests/ci/README.md @@ -96,6 +96,14 @@ their own refs. Independent groups do not enforce PR creation order. The require gate still rejects failed, cancelled or missing results. Manual cancellation can still prevent the `always()` summary from completing. See [GitHub concurrency rules](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#concurrency). +The exact root `AGENTS.md` path also qualifies as documentation. A known, +nonempty range confined to that file and other recognized documentation selects +Nimbus and the final required gate on both PRs and merge-group candidates, with +native jobs explicitly skipped when the site is available. This does not extend +the root README link check or exempt nested `AGENTS.md` files, similarly named +code, mixed source changes or uncertain ranges. Manual runs retain the full +native matrix. This CI policy change itself still requires native acceptance. + Within `src/`, only the exact plugin index paths `src/plugins/README.md` and `src/plugins/README_CN.md` additionally qualify as documentation. Changes confined to these indexes select the Nimbus check on PRs and documentation-only queue diff --git a/tests/ci/README_CN.md b/tests/ci/README_CN.md index f78798d0f..9ee8db541 100644 --- a/tests/ci/README_CN.md +++ b/tests/ci/README_CN.md @@ -66,6 +66,12 @@ README 徽章展示合并候选的工作流状态。 required gate 仍拒绝失败、取消或缺失的结果。手动取消仍可能使 `always()` 汇总无法完成。 详见 [GitHub 并发规则](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#concurrency)。 +根目录的精确路径 `AGENTS.md` 也归为文档。范围已知且非空、文档站可用时， +仅修改该文件和其他已识别文档的 PR 与合并候选均要求 Nimbus 检查和最终必需门禁， +并明确跳过原生任务。这不会扩展根 README 链接检查，也不会豁免嵌套的 `AGENTS.md`、 +名称相近的代码、混合源码变更或不确定范围。手动运行仍保留完整原生矩阵。 +本次 CI 策略变更自身仍须完成原生验收。 + `src/` 下仅将两个精确的插件索引路径 `src/plugins/README.md` 与 `src/plugins/README_CN.md` 归为文档。 文档站可用时，只修改这两个索引的 PR 与纯文档合并候选会选择 Nimbus 检查。 这不会将根 README 链接检查扩展到插件索引；Nimbus 验证生成的文档站，不直接检查索引文件。 diff --git a/tests/ci/module-validation.test.mjs b/tests/ci/module-validation.test.mjs index fe96edd87..1fa8aaf95 100644 --- a/tests/ci/module-validation.test.mjs +++ b/tests/ci/module-validation.test.mjs @@ -30,7 +30,8 @@ test('actual workflow output bindings provide complete plans to the required gat const outputs = [...jobBlock('scope').matchAll(/^ (\w+): \$\{\{ steps\.scope\.outputs\.(\w+) \}\}$/gm)]; assert.ok(outputs.length > 0); for (const scope of [plan(['tests/network/rudp.cpp']), plan(['include/ark/core/module.hpp']), - plan(['README.md']), plan(['tests/database/redis_service.cpp'], 'merge_group'), + plan(['README.md']), plan(['AGENTS.md']), plan(['AGENTS.md'], 'merge_group'), + plan(['tests/database/redis_service.cpp'], 'merge_group'), plan(['README.md'], 'merge_group'), plan([], 'workflow_dispatch'), plan([], 'pull_request')]) { const needs = results(scope); needs.scope.outputs = Object.fromEntries(outputs.map(([, name, source]) => [name, String(scope[source])])); @@ -113,7 +114,7 @@ test('public core headers, build/dependency/CI inputs and unknown paths retain e '3rdparty/spdlog', 'ports/asio/portfile.cmake', '.github/workflows/ci.yml', 'config/timers.app.toml', 'resources/excel/item.xlsx', 'resources/photon/proto/protocol.proto', 'tools/gen-config.sh', 'deployment/scripts/run.sh', - '.github/scripts/ci-policy.mjs', 'tests/ci/module-validation.test.mjs', 'AGENTS.md', '.clang-format', + '.github/scripts/ci-policy.mjs', 'tests/ci/module-validation.test.mjs', 'src/AGENTS.md', '.clang-format', 'application/unknown.bin', 'application/build.cmake', 'applications/main.cpp', 'src/app/main.cpp', 'src/plugins/new_plugin/new_module.cpp', 'src/plugins/net/unknown.bin', 'scripts/new-check.sh', 'docs/example.cpp', 'docs/site/package-lock.json', 'docs/unknown.bin', 'LICENSE']) { diff --git a/tests/ci/policy.test.mjs b/tests/ci/policy.test.mjs index 2a0810544..46c3bdbc8 100644 --- a/tests/ci/policy.test.mjs +++ b/tests/ci/policy.test.mjs @@ -64,6 +64,99 @@ test('only recognized documentation paths qualify for the lightweight plan', () assert.equal(decision(['README.md', 'docs/guide.md']).docs_check, true); }); +test('root AGENTS.md requires documentation and the final gate without native work at both scoped stages', () => { + assert.equal(isDocumentationPath('AGENTS.md'), true); + for (const eventName of ['pull_request', 'merge_group']) { + for (const files of [['AGENTS.md'], ['AGENTS.md', 'docs/guide.md']]) { + const scope = decision(files, { eventName }); + assert.equal(scope.validation_stage, 'documentation'); + assert.equal(scope.reason, 'documentation-only'); + assert.equal(scope.docs_check, true); + assert.equal(scope.readme_check, false); + for (const key of ['native_check', 'format_check', 'network_check', 'applications_check', 'database_check']) { + assert.equal(scope[key], false, `${eventName}/${key}`); + } + validateGate(results(scope)); + for (const outcome of ['skipped', 'failure', 'cancelled', undefined]) { + const invalid = results(scope); + invalid.validation.outputs.documentation_outcome = outcome; + assert.throws(() => validateGate(invalid), `${eventName}/${outcome}`); + } + for (const result of ['skipped', 'failure', 'cancelled', undefined]) { + const invalid = results(scope); + invalid.validation.result = result; + assert.throws(() => validateGate(invalid), `${eventName}/validation/${result}`); + } + } + assert.equal(decision(['AGENTS.md', 'README.md'], { eventName }).readme_check, true); + } +}); + +test('an exact root AGENTS.md Git change selects documentation through the real PR and merge-group classifier', t => { + const repo = repository(t); + const base = addSite(repo); + repo.write('AGENTS.md', '# Contributor and agent instructions\n'); + const head = repo.commit(); + assert.deepEqual(changedPaths({ root: repo.root, base, head }), ['AGENTS.md']); + for (const eventName of ['pull_request', 'merge_group']) { + const payload = eventName === 'pull_request' ? { pull_request: { base: { sha: base }, head: { sha: head } } } : + { merge_group: { base_sha: base, head_sha: head } }; + const { plan: scope, output } = classifyEvent(repo, eventName, payload); + assert.equal(scope.validation_stage, 'documentation'); + assert.equal(scope.docs_check, true); + assert.equal(scope.readme_check, false); + for (const key of ['native_check', 'format_check', 'network_check', 'applications_check', 'database_check']) { + assert.equal(scope[key], false, `${eventName}/${key}`); + assert.match(output, new RegExp(`^${key}=false$`, 'm')); + } + validateGate(results(scope)); + } +}); + +test('AGENTS neighbors and nested unknown paths cannot inherit the root documentation exception', () => { + const paths = ['agents.md', 'AGENTS.MD', 'AGENTS_CN.md', 'AGENTS.md.bak', 'AGENTS.md.js', + 'AGENTS.cpp', 'AGENTS.md.hpp', 'nested/AGENTS.md', 'src/AGENTS.md', 'tests/AGENTS.md', + 'src/plugins/net/AGENTS.md', '.github/AGENTS.md', 'docs/AGENTS.cpp', './AGENTS.md', + 'nested/../AGENTS.md', 'nested//AGENTS.md', 'nested\\AGENTS.md']; + for (const file of paths) { + assert.equal(isDocumentationPath(file), false, file); + for (const eventName of ['pull_request', 'merge_group']) { + const scope = decision(['AGENTS.md', file], { eventName }); + assert.equal(scope.native_check, true, `${eventName}/${file}`); + assert.equal(scope.format_check, true); + assert.equal(scope.docs_check, true); + for (const job of ['network', 'applications', 'database']) assert.equal(scope[`${job}_check`], true); + validateGate(results(scope)); + } + } +}); + +test('AGENTS mixed with code keeps source module selection and every merge-candidate native module', () => { + for (const eventName of ['pull_request', 'merge_group']) { + const scope = decision(['AGENTS.md', 'tests/core/containers.cpp'], { eventName }); + assert.equal(scope.native_check, true); + assert.equal(scope.docs_check, true); + assert.equal(scope.format_check, true); + assert.equal(scope.validation_stage, eventName === 'pull_request' ? 'modules' : 'full'); + for (const job of ['network', 'applications', 'database']) { + assert.equal(scope[`${job}_check`], eventName !== 'pull_request' || job === 'applications'); + } + validateGate(results(scope)); + } +}); + +test('AGENTS never exempts manual events, unavailable sites or uncertain changed paths from native checks', () => { + for (const extra of [{ eventName: 'workflow_dispatch' }, { enumerationFailed: true }, + { site: { present: true, available: false } }, { site: { present: false, available: false } }]) { + const scope = decision(['AGENTS.md'], extra); + assert.equal(scope.native_check, true); + assert.equal(scope.validation_stage, 'full'); + for (const job of ['network', 'applications', 'database']) assert.equal(scope[`${job}_check`], true); + if (extra.site?.present) assert.equal(scope.docs_check, true); + validateGate(results(scope)); + } +}); + test('only the two exact plugin indexes select documentation and keep root README checks separate', () => { const indexes = ['src/plugins/README.md', 'src/plugins/README_CN.md']; for (const eventName of ['pull_request', 'merge_group']) {