From b8b1e611a798f957b0fbd3ca9887ef6e5ed6d190 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 18 Nov 2026 18:00:20 -0600 Subject: [PATCH 1/2] Keep curl teardown from hanging on stalled DNS When the collector's DNS is unreachable, teardown of the curl transport could block host process exit for the system resolver timeout (tens of seconds). curl_easy_perform() does not run the progress callback while a name is resolving, so Abort() waited for the 5s connect timeout, and curl_easy_cleanup()/curl_multi_cleanup() then joined the resolver thread still blocked in getaddrinfo(). Set CURLOPT_QUICK_EXIT on every handle, and run modern-libcurl transfers on a private multi handle with CURLMOPT_QUICK_EXIT, polling the abort flag every 100ms. Option ids are passed numerically so a runtime libcurl newer than the build headers still receives them. A Linux unit test interposes getaddrinfo() to stall deterministically; it hangs 60s without this change. Files changed: - lib/http/HttpClient_Curl.hpp - tests/unittests/CMakeLists.txt - tests/unittests/HttpClientCurlTests.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ff620832-ec83-4315-b3e0-6f5d825d45e1 --- lib/http/HttpClient_Curl.hpp | 86 +++++++++++++++++- tests/unittests/CMakeLists.txt | 4 + tests/unittests/HttpClientCurlTests.cpp | 114 ++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 1 deletion(-) diff --git a/lib/http/HttpClient_Curl.hpp b/lib/http/HttpClient_Curl.hpp index eac40ad13..c5d6bf7b6 100644 --- a/lib/http/HttpClient_Curl.hpp +++ b/lib/http/HttpClient_Curl.hpp @@ -45,6 +45,18 @@ #define HTTP_CONN_TIMEOUT 5L +// libcurl option ids are ABI-stable and never reused. Pass these by number so +// a runtime libcurl newer than the build headers still receives them; an +// older runtime rejects an unknown id without side effects. +#define MATSDK_CURLOPT_QUICK_EXIT static_cast(CURLOPTTYPE_LONG + 322) // libcurl 7.87.0 +#define MATSDK_CURLMOPT_QUICK_EXIT static_cast(CURLOPTTYPE_LONG + 21) // libcurl 8.20.0 +#if LIBCURL_VERSION_NUM >= 0x075700 +static_assert(MATSDK_CURLOPT_QUICK_EXIT == CURLOPT_QUICK_EXIT, "CURLOPT_QUICK_EXIT id changed"); +#endif +#if LIBCURL_VERSION_NUM >= 0x081400 +static_assert(MATSDK_CURLMOPT_QUICK_EXIT == CURLMOPT_QUICK_EXIT, "CURLMOPT_QUICK_EXIT id changed"); +#endif + #undef TRACE #define TRACE(...) // printf @@ -337,6 +349,14 @@ class CurlHttpOperation { // With NOSIGNAL, a synchronous resolver may still prevent libcurl from // enforcing a strict deadline until the resolver call returns. + // Teardown drains every operation, and curl_easy_cleanup() otherwise + // joins a resolver thread that can stay blocked in getaddrinfo() for the + // system DNS timeout when the collector is unreachable, stalling host + // process exit. Best effort: an older runtime libcurl rejects it. + // Trade-off: a resolver thread abandoned this way runs until + // getaddrinfo() returns, and must not outlive an unloaded libcurl. + (void)curl_easy_setopt(curl, MATSDK_CURLOPT_QUICK_EXIT, 1L); + // Headers are copied into m_headersChunk during construction and the // curl_slist is kept alive until destruction, so the original map does // not need operation-lifetime storage. @@ -551,7 +571,8 @@ class CurlHttpOperation { goto cleanup; } DispatchEvent(m_usePrereqCallback ? OnConnecting : OnSending); - m_transportError = curl_easy_perform(curl); + // The legacy CONNECT_ONLY path must reuse the easy handle's connection. + m_transportError = m_usePrereqCallback ? PerformAbortable() : curl_easy_perform(curl); if(CURLE_OK != m_transportError) { DispatchEvent(!m_usePrereqCallback || m_connectionReady ? OnSendFailed : OnConnectFailed); @@ -961,6 +982,69 @@ class CurlHttpOperation { } } + /** + * Run the transfer on a private multi handle so cancellation is observed + * within one poll slice. curl_easy_perform() does not invoke the progress + * callback while a name is resolving, so an abort during a stalled DNS + * lookup would otherwise wait for the full connect timeout. + */ + CURLcode PerformAbortable() + { +#if LIBCURL_VERSION_NUM >= 0x074200 // libcurl 7.66.0: curl_multi_poll + CURLM* multi = curl_multi_init(); + if (multi == nullptr) + { + return CURLE_OUT_OF_MEMORY; + } + // From libcurl 8.20.0 resolver threads belong to the multi handle; do + // not join them. + (void)curl_multi_setopt(multi, MATSDK_CURLMOPT_QUICK_EXIT, 1L); + if (curl_multi_add_handle(multi, curl) != CURLM_OK) + { + curl_multi_cleanup(multi); + return CURLE_FAILED_INIT; + } + + CURLcode result = CURLE_FAILED_INIT; + constexpr int sliceMs = 100; + for (;;) + { + int running = 0; + if (curl_multi_perform(multi, &running) != CURLM_OK) + { + break; + } + if (running == 0) + { + int queued = 0; + while (CURLMsg* msg = curl_multi_info_read(multi, &queued)) + { + if (msg->msg == CURLMSG_DONE && msg->easy_handle == curl) + { + result = msg->data.result; + } + } + break; + } + if (isAborted.load(std::memory_order_acquire)) + { + result = CURLE_ABORTED_BY_CALLBACK; + break; + } + if (curl_multi_poll(multi, nullptr, 0, sliceMs, nullptr) != CURLM_OK) + { + break; + } + } + + curl_multi_remove_handle(multi, curl); + curl_multi_cleanup(multi); + return result; +#else + return curl_easy_perform(curl); +#endif + } + /** * Install the libcurl progress callback used to abort a transfer. * diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index ff682427b..d2c6c6c96 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -149,6 +149,10 @@ target_link_libraries(UnitTests PRIVATE ZLIB::ZLIB gtest gmock) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # HttpClientCurlTests interposes getaddrinfo() and forwards via dlsym(). + target_link_libraries(UnitTests PRIVATE ${CMAKE_DL_LIBS}) +endif() if(NOT MATSDK_SQLITE_PROVIDER_RESOLVED STREQUAL "NONE") target_link_libraries(UnitTests PRIVATE SQLite3::SQLite3) endif() diff --git a/tests/unittests/HttpClientCurlTests.cpp b/tests/unittests/HttpClientCurlTests.cpp index 1d6236e19..4b964b7bc 100644 --- a/tests/unittests/HttpClientCurlTests.cpp +++ b/tests/unittests/HttpClientCurlTests.cpp @@ -31,6 +31,8 @@ #include #include +#include +#include #include #include #include @@ -1239,4 +1241,116 @@ TEST_F(HttpClientCurlLifetimeTests, OverlappingClientsWithActiveRequestsDestroyI } } +#if defined(__linux__) + +// An unreachable DNS server keeps getaddrinfo() blocked for the system resolver +// timeout. Interpose it for one reserved host so the stall is deterministic and +// needs no network; every other lookup goes to the real resolver. +namespace +{ +constexpr char kStalledHost[] = "dns-stall.invalid"; + +// Each test run is one generation, so a resolver thread detached by an earlier +// run (for example under --gtest_repeat) keeps waiting on its own release. +struct DnsStall +{ + std::mutex mutex; + std::condition_variable cv; + uint64_t generation {0}; + uint64_t entered {0}; + uint64_t released {0}; +}; + +DnsStall& GetDnsStall() +{ + static DnsStall* stall = new DnsStall(); // Leaked: a detached resolver may still use it at exit. + return *stall; +} + +uint64_t BeginDnsStall() +{ + DnsStall& stall = GetDnsStall(); + std::lock_guard lock(stall.mutex); + return ++stall.generation; +} + +void ReleaseDnsStall(uint64_t generation) +{ + DnsStall& stall = GetDnsStall(); + { + std::lock_guard lock(stall.mutex); + stall.released = generation; + } + stall.cv.notify_all(); +} +} // namespace + +extern "C" int getaddrinfo(const char* node, const char* service, const struct addrinfo* hints, + struct addrinfo** res) +{ + if (node != nullptr && std::strcmp(node, kStalledHost) == 0) + { + DnsStall& stall = GetDnsStall(); + std::unique_lock lock(stall.mutex); + const uint64_t generation = stall.generation; + stall.entered = generation; + stall.cv.notify_all(); + stall.cv.wait_for(lock, std::chrono::seconds(60), [&]() { return stall.released >= generation; }); + return EAI_AGAIN; + } + using GetAddrInfoFn = int (*)(const char*, const char*, const struct addrinfo*, struct addrinfo**); + static const auto real = reinterpret_cast(dlsym(RTLD_NEXT, "getaddrinfo")); + return (real != nullptr) ? real(node, service, hints, res) : EAI_SYSTEM; +} + +// Destroying the client while name resolution is stalled must abort and drain +// promptly: neither the abort nor curl_easy_cleanup() may wait for the resolver. +TEST(HttpClientCurlDnsStallTests, DestroyingClientDuringStalledResolutionDoesNotHang) +{ + const curl_version_info_data* versionInfo = curl_version_info(CURLVERSION_NOW); + if (!UsesPrereqCallback() || versionInfo == nullptr || versionInfo->version_num < 0x075700) + { + GTEST_SKIP() << "runtime libcurl predates CURLOPT_QUICK_EXIT (7.87.0)"; + } + if ((versionInfo->features & CURL_VERSION_ASYNCHDNS) == 0) + { + // A synchronous resolver blocks the worker itself; nothing can abort it. + GTEST_SKIP() << "runtime libcurl resolves names synchronously"; + } + + const uint64_t generation = BeginDnsStall(); + RecordingCallback callback; + std::unique_ptr client(new HttpClient_Curl()); + std::unique_ptr request(client->CreateRequest()); + request->SetUrl(std::string("http://") + kStalledHost + "/stall"); + client->SendRequestAsync(request.get(), &callback); + + bool entered = false; + { + DnsStall& stall = GetDnsStall(); + std::unique_lock lock(stall.mutex); + entered = stall.cv.wait_for(lock, std::chrono::seconds(10), [&]() { return stall.entered == generation; }); + } + if (!entered) + { + client.reset(); + ReleaseDnsStall(generation); + GTEST_SKIP() << "libcurl does not resolve through getaddrinfo() (for example, c-ares)"; + } + + const auto start = std::chrono::steady_clock::now(); + client.reset(); + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + ReleaseDnsStall(generation); + + // The connect timeout is 5s; finishing well inside it proves the abort did + // not wait for it, and no resolver join extended it to the 60s stall. + EXPECT_LT(elapsed.count(), 3000) << "client teardown waited on the stalled resolver"; + EXPECT_EQ(callback.responses(), 1u); + EXPECT_EQ(callback.responsesWithResult(HttpResult_Aborted), 1u); +} + +#endif // __linux__ + #endif // MATSDK_PAL_CPP11 && !_MSC_VER && HAVE_MAT_DEFAULT_HTTP_CLIENT From d71a6072e4bdb6a6af23f2064f75c52a533d1583 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 19 Nov 2026 11:49:17 -0600 Subject: [PATCH 2/2] Keep CI checks reliable on newer runner images Include the integer type dependency in ICdsFactory.hpp and avoid the racy mkdir implementation in build-runner-tests.py. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ea7f5226-1c17-476f-abaa-90cfa4b32e3c --- lib/include/public/ICdsFactory.hpp | 1 + tests/build-runner-tests.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/include/public/ICdsFactory.hpp b/lib/include/public/ICdsFactory.hpp index 312c3b210..f5906969e 100644 --- a/lib/include/public/ICdsFactory.hpp +++ b/lib/include/public/ICdsFactory.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/tests/build-runner-tests.py b/tests/build-runner-tests.py index 8360c1142..7a64c8e04 100644 --- a/tests/build-runner-tests.py +++ b/tests/build-runner-tests.py @@ -33,7 +33,7 @@ def setUp(self): self.write_script( "out/tests/functests/FuncTests", 'printf "concurrent %s\\n" "$*" >> "$CALLS"\n' - 'if mkdir "$CASE_ROOT/first-child" 2>/dev/null; then\n' + 'if (set -C; : > "$CASE_ROOT/first-child") 2>/dev/null; then\n' ' sleep 0.05\n' ' echo first-finished >> "$CALLS"\n' ' exit "${FIRST_RESULT:-0}"\n' @@ -123,6 +123,13 @@ def test_second_child_failure_is_not_lost(self): self.assertIn("first-finished", calls) self.assertIn("second-finished", calls) + def test_child_selection_does_not_depend_on_mkdir(self): + self.write_script("bin/mkdir", "exit 0\n") + result, calls = self.run_runner(SECOND_RESULT="26") + self.assertEqual(result.returncode, 26, result.stdout + result.stderr) + self.assertIn("first-finished", calls) + self.assertIn("second-finished", calls) + if __name__ == "__main__": unittest.main()