diff --git a/lib/offline/OfflineStorageHandler.cpp b/lib/offline/OfflineStorageHandler.cpp index 60f67fb64..ec6f5de84 100644 --- a/lib/offline/OfflineStorageHandler.cpp +++ b/lib/offline/OfflineStorageHandler.cpp @@ -12,6 +12,7 @@ #include "ILogManager.hpp" #include "utils/Utils.hpp" #include +#include #include #include #include @@ -32,6 +33,152 @@ namespace MAT_NS_BEGIN { MATSDK_LOG_INST_COMPONENT_CLASS(OfflineStorageHandler, "EventsSDK.StorageHandler", "Events telemetry client - OfflineStorageHandler class") + // Queued tasks retain this state even after the handler has stopped. + class OfflineStorageHandler::FlushState + { + public: + enum class Phase { Accepting, Draining, Closing, Retryable, Stopped }; + enum class ShutdownStage { Flush, MemoryShutdown, DiskShutdown, Complete }; + + struct PendingTask + { + // Keep the task alive while cancellation looks up its identity. + std::recursive_mutex mutex; + Task* task = nullptr; + }; + + explicit FlushState(OfflineStorageHandler* handler) : owner(handler) {} + + std::mutex mutex; + std::condition_variable cv; + Phase phase = Phase::Accepting; + ShutdownStage shutdownStage = ShutdownStage::Flush; + OfflineStorageHandler* owner; + size_t active = 0; + std::shared_ptr pending; + }; + + class OfflineStorageHandler::OperationGuard + { + public: + explicit OperationGuard(std::shared_ptr state, bool allowDraining = false) : + m_state(std::move(state)) + { + std::lock_guard lock(m_state->mutex); + if (m_state->phase == FlushState::Phase::Accepting || + (allowDraining && m_state->phase == FlushState::Phase::Draining)) + { + m_owner = m_state->owner; + ++m_state->active; + } + } + + OperationGuard(std::shared_ptr state, + std::shared_ptr const& pending) : + m_state(std::move(state)) + { + std::lock_guard lock(m_state->mutex); + if (m_state->pending != pending) + { + return; + } + m_state->pending.reset(); + if (m_state->phase == FlushState::Phase::Accepting) + { + m_owner = m_state->owner; + ++m_state->active; + } + } + + ~OperationGuard() noexcept + { + if (m_owner) + { + { + std::lock_guard lock(m_state->mutex); + --m_state->active; + } + m_state->cv.notify_all(); + } + } + + explicit operator bool() const noexcept { return m_owner != nullptr; } + OfflineStorageHandler* Owner() const noexcept { return m_owner; } + + OperationGuard(OperationGuard const&) = delete; + OperationGuard& operator=(OperationGuard const&) = delete; + + private: + std::shared_ptr m_state; + OfflineStorageHandler* m_owner = nullptr; + }; + + class OfflineStorageHandler::ScheduledFlushTask final : public Task + { + public: + ScheduledFlushTask(std::shared_ptr state, + std::shared_ptr pending) : + m_state(std::move(state)), + m_pending(std::move(pending)) + { + Type = Task::Call; + TargetTime = 0; + TypeName = "OfflineStorageFlushTask"; + std::lock_guard lock(m_pending->mutex); + m_pending->task = this; + } + + ~ScheduledFlushTask() noexcept override + { + { + std::lock_guard lock(m_pending->mutex); + m_pending->task = nullptr; + } + std::lock_guard lock(m_state->mutex); + if (m_state->pending == m_pending) + { + m_state->pending.reset(); + } + } + + void operator()() override + { + OperationGuard operation(m_state, m_pending); + if (operation) + { + operation.Owner()->Flush(); + } + } + + private: + std::shared_ptr m_state; + std::shared_ptr m_pending; + }; + + class OfflineStorageHandler::TeardownGuard + { + public: + explicit TeardownGuard(std::shared_ptr state) : m_state(std::move(state)) {} + + ~TeardownGuard() noexcept + { + { + std::lock_guard lock(m_state->mutex); + m_state->phase = m_completed ? FlushState::Phase::Stopped : FlushState::Phase::Retryable; + } + m_state->cv.notify_all(); + } + + void Complete() noexcept { m_completed = true; } + + TeardownGuard(TeardownGuard const&) = delete; + TeardownGuard& operator=(TeardownGuard const&) = delete; + + private: + std::shared_ptr m_state; + bool m_completed = false; + }; + OfflineStorageHandler::OfflineStorageHandler(ILogManager& logManager, IRuntimeConfig& runtimeConfig, ITaskDispatcher& taskDispatcher) : OfflineStorageHandler(logManager, runtimeConfig, taskDispatcher, OfflineStorageFactory::GetDefaultProvider()) { @@ -45,7 +192,7 @@ namespace MAT_NS_BEGIN { m_storageProvider(std::move(storageProvider)), m_killSwitchManager(), m_clockSkewManager(), - m_flushPending(false), + m_flushState(std::make_shared(this)), m_offlineStorageMemory(nullptr), m_offlineStorageDisk(nullptr), m_readFromMemory(false), @@ -136,33 +283,70 @@ namespace MAT_NS_BEGIN { /* slower */ m_killSwitchManager.isTokenBlocked(record.tenantToken)); } - void OfflineStorageHandler::WaitForFlush() + bool OfflineStorageHandler::BeginTeardown() { + auto state = m_flushState; + std::shared_ptr pending; { - LOCKGUARD(m_flushLock); - if (!m_flushPending) - return; + std::unique_lock lock(state->mutex); + state->cv.wait(lock, [&state] { + return state->phase != FlushState::Phase::Draining && + state->phase != FlushState::Phase::Closing; + }); + if (state->phase == FlushState::Phase::Stopped) + { + return false; + } + if (state->phase == FlushState::Phase::Retryable) + { + state->phase = FlushState::Phase::Closing; + return true; + } + // Drain admitted operations before closing storage or invalidating queued work. + m_shutdownStarted = true; + state->phase = FlushState::Phase::Draining; + state->cv.wait(lock, [&state] { return state->active == 0; }); + state->phase = FlushState::Phase::Closing; + state->owner = nullptr; + pending = std::move(state->pending); + } + + if (pending) + { + MATSDK_TRY + { + std::lock_guard lock(pending->mutex); + if (pending->task) + { + m_taskDispatcher.Cancel(pending->task); + } + } + MATSDK_CATCH(...) + { + std::fputs("Failed to cancel pending offline flush\n", stderr); + } } - LOG_INFO("Waiting for pending Flush (%p) to complete...", - static_cast(m_flushHandle.GetTask())); - m_flushComplete.wait(); + return true; } OfflineStorageHandler::~OfflineStorageHandler() { - WaitForFlush(); - if (nullptr != m_offlineStorageMemory) + if (BeginTeardown()) { + TeardownGuard teardown(m_flushState); m_offlineStorageMemory.reset(); - } - if (nullptr != m_offlineStorageDisk) - { m_offlineStorageDisk.reset(); + teardown.Complete(); } } void OfflineStorageHandler::Initialize(IOfflineStorageObserver& observer) { + Shutdown(); + m_offlineStorageMemory.reset(); + m_offlineStorageDisk.reset(); + m_flushState = std::make_shared(this); + m_shutdownStarted = false; m_observer = &observer; m_cacheMemorySizeLimitInBytes = m_config[CFG_INT_RAM_QUEUE_SIZE]; @@ -181,25 +365,48 @@ namespace MAT_NS_BEGIN { m_offlineStorageMemory->Initialize(*this); } - m_shutdownStarted = false; LOG_TRACE("Initializing offline storage handler"); } void OfflineStorageHandler::Shutdown() { LOG_TRACE("Shutting down offline storage handler"); - m_shutdownStarted = true; - WaitForFlush(); - if (nullptr != m_offlineStorageMemory) + if (!BeginTeardown()) { - m_offlineStorageMemory->ReleaseAllRecords(); - Flush(); - m_offlineStorageMemory->Shutdown(); + return; } - if (nullptr != m_offlineStorageDisk) + auto state = m_flushState; + TeardownGuard teardown(state); + // A failed teardown retries only the stage that did not complete. + if (state->shutdownStage == FlushState::ShutdownStage::Flush) { - m_offlineStorageDisk->Shutdown(); + if (nullptr != m_offlineStorageMemory) + { + m_offlineStorageMemory->ReleaseAllRecords(); + { + ActivityGuard activity(m_logManager, true); + FlushImpl(); + } + } + state->shutdownStage = FlushState::ShutdownStage::MemoryShutdown; } + if (state->shutdownStage == FlushState::ShutdownStage::MemoryShutdown) + { + if (nullptr != m_offlineStorageMemory) + { + m_offlineStorageMemory->Shutdown(); + } + state->shutdownStage = FlushState::ShutdownStage::DiskShutdown; + } + if (state->shutdownStage == FlushState::ShutdownStage::DiskShutdown) + { + if (nullptr != m_offlineStorageDisk) + { + m_offlineStorageDisk->Shutdown(); + } + state->shutdownStage = FlushState::ShutdownStage::Complete; + } + teardown.Complete(); } /// @@ -243,33 +450,30 @@ namespace MAT_NS_BEGIN { void OfflineStorageHandler::Flush() { - // Shutdown has already paused normal logging, but its synchronous final - // flush must still persist the in-memory records before storage closes. - ActivityGuard activityGuard(m_logManager, m_shutdownStarted); + OperationGuard operation(m_flushState); + if (!operation) + { + return; + } + ActivityGuard activityGuard(m_logManager); if (!activityGuard.IsActive()) { - // The LogManager is shutting down, so the flush cannot run. Still - // signal completion and clear the pending flag so a concurrent - // WaitForFlush() (e.g. during teardown) does not block forever - // waiting for m_flushComplete. - LOCKGUARD(m_flushLock); - m_flushHandle.Cancel(); - m_flushComplete.post(); - m_flushPending = false; return; } + FlushImpl(); + } + + void OfflineStorageHandler::FlushImpl() + { std::vector recordsToRecover; std::vector memoryOnlyRecords; + size_t savedRecords = 0; + bool notifySaved = false; MATSDK_TRY { // Flush could be executed from context of worker thread, as well as from TPM and // after HTTP callback. Make sure it is atomic / thread-safe. LOCKGUARD(m_flushLock); - // If item isn't scheduled yet, it gets canceled, so that we don't do two flushes. - // If we are running that item right now (our thread), then nothing happens other - // than the handle reporting nullptr once that task finishes. - m_flushHandle.Cancel(); - size_t dbSizeBeforeFlush = (m_offlineStorageMemory != nullptr) ? m_offlineStorageMemory->GetSize() : 0; if ((m_offlineStorageMemory) && (dbSizeBeforeFlush > 0) && (m_offlineStorageDisk)) { @@ -316,9 +520,14 @@ namespace MAT_NS_BEGIN { }), recordsToRecover.end()); - const size_t batchSaved = recordsToRecover.empty() - ? 0 - : m_offlineStorageDisk->StoreRecords(recordsToRecover); + size_t batchSaved = 0; + if (!recordsToRecover.empty()) + { + // Storage can consume its input, so retain the original batch for retry. + auto recordsForStorage = recordsToRecover; + batchSaved = m_offlineStorageDisk->StoreRecords(recordsForStorage); + } + totalSaved += std::min(batchSaved, recordsToRecover.size()); if (batchSaved != recordsToRecover.size()) { LOG_WARN("Flush: disk store failed for the batch of %zu records; returning it to the queue for retry", @@ -328,7 +537,6 @@ namespace MAT_NS_BEGIN { break; } - totalSaved += batchSaved; recordsToRecover.clear(); } ReturnRecordsToMemory(memoryOnlyRecords); @@ -353,7 +561,8 @@ namespace MAT_NS_BEGIN { // obviously because the disk is slower than ram. LOG_WARN("Data is arriving too fast!"); } - OnStorageRecordsSaved(totalSaved); + savedRecords = totalSaved; + notifySaved = true; } // Checkpoint DB @@ -363,8 +572,6 @@ namespace MAT_NS_BEGIN { } m_isStorageFullNotificationSend = false; - m_flushComplete.post(); - m_flushPending = false; } MATSDK_CATCH(...) { @@ -385,16 +592,65 @@ namespace MAT_NS_BEGIN { { std::fputs("Failed to recover records after flush failure\n", stderr); } - LOCKGUARD(m_flushLock); - m_flushComplete.post(); - m_flushPending = false; + if (notifySaved) + { + MATSDK_TRY + { + OnStorageRecordsSaved(savedRecords); + } + MATSDK_CATCH(...) + { + std::fputs("Failed to report saved records after flush failure\n", stderr); + } + } std::rethrow_exception(failure); +#endif + } + if (notifySaved) + { + OnStorageRecordsSaved(savedRecords); + } + } + + void OfflineStorageHandler::QueueScheduledFlush() + { + auto state = m_flushState; + std::shared_ptr pending; + { + std::lock_guard lock(state->mutex); + if (state->phase != FlushState::Phase::Accepting || state->pending) + { + return; + } + pending = std::make_shared(); + state->pending = pending; + } + + MATSDK_TRY + { + std::unique_ptr task(new ScheduledFlushTask(state, pending)); + m_taskDispatcher.Queue(task.release()); + } + MATSDK_CATCH(...) + { +#if HAVE_EXCEPTIONS + std::lock_guard lock(state->mutex); + if (state->pending == pending) + { + state->pending.reset(); + } + throw; #endif } } bool OfflineStorageHandler::StoreRecord(StorageRecord const& record) { + OperationGuard operation(m_flushState, true); + if (!operation) + { + return false; + } // Don't discard on shutdown because the kill-switch may be temporary. // Attempt to upload after restart. if ((!m_shutdownStarted) && isKilled(record)) @@ -436,25 +692,7 @@ namespace MAT_NS_BEGIN { // Perform periodic flush to disk if (memDbSize > cacheMemorySizeLimitInBytes) { - if (m_flushLock.try_lock()) - { - if (!m_flushPending) - { - m_flushPending = true; - m_flushComplete.Reset(); - m_flushHandle = PAL::scheduleTask(&m_taskDispatcher, 0, this, &OfflineStorageHandler::Flush); - if (m_flushHandle.GetTask() == nullptr) - { - // The dispatcher may drop a task synchronously during - // shutdown. Do not leave WaitForFlush blocked forever. - m_flushPending = false; - m_flushComplete.post(); - } - LOG_INFO("Requested Flush (%p)", - static_cast(m_flushHandle.GetTask())); - } - m_flushLock.unlock(); - } + QueueScheduledFlush(); } } else diff --git a/lib/offline/OfflineStorageHandler.hpp b/lib/offline/OfflineStorageHandler.hpp index 0eae7510b..3c69640be 100644 --- a/lib/offline/OfflineStorageHandler.hpp +++ b/lib/offline/OfflineStorageHandler.hpp @@ -14,9 +14,10 @@ #include "ILogManager.hpp" #include "pal/TaskDispatcher.hpp" -#include #include #include +#include +#include #include #include "KillSwitchManager.hpp" @@ -82,9 +83,11 @@ namespace MAT_NS_BEGIN { bool isKilled(StorageRecord const& record); std::mutex m_flushLock; - bool m_flushPending; - PAL::DeferredCallbackHandle m_flushHandle; - PAL::Event m_flushComplete; + class FlushState; + class OperationGuard; + class ScheduledFlushTask; + class TeardownGuard; + std::shared_ptr m_flushState; std::shared_ptr m_offlineStorageMemory; std::shared_ptr m_offlineStorageDisk; @@ -92,7 +95,7 @@ namespace MAT_NS_BEGIN { std::atomic m_readFromMemory; std::atomic m_lastReadCount; - bool m_shutdownStarted; + std::atomic m_shutdownStarted; unsigned m_memoryDbSize; unsigned m_memoryDbSizeNotificationLimit; unsigned m_queryDbSize; @@ -103,7 +106,9 @@ namespace MAT_NS_BEGIN { MATSDK_LOG_DECL_COMPONENT_CLASS(); private: - void WaitForFlush(); + bool BeginTeardown(); + void QueueScheduledFlush(); + void FlushImpl(); bool IsBatchedStorageFlushEnabled(); void ReportInvalidDiskRecord(StorageRecord const& record); size_t StoreRecordsIndividually(std::vector& records); diff --git a/lib/pal/WorkerThread.cpp b/lib/pal/WorkerThread.cpp index 03878f298..46da4f62f 100644 --- a/lib/pal/WorkerThread.cpp +++ b/lib/pal/WorkerThread.cpp @@ -295,19 +295,15 @@ namespace PAL_NS_BEGIN { queuedItem = *it; m_timerQueue.erase(it); } - lock.unlock(); - delete queuedItem; -#if 0 - for (;;) { - { - LOCKGUARD(m_lock); - if (item->Type == MAT::Task::Done) { - return; - } + else { + auto immediate = std::find(m_queue.begin(), m_queue.end(), item); + if (immediate != m_queue.end()) { + queuedItem = *immediate; + m_queue.erase(immediate); } - Sleep(10); } -#endif + lock.unlock(); + delete queuedItem; return true; } diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index ff682427b..e2204546c 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -68,6 +68,9 @@ set(MATSDK_UNIT_TEST_DEFINITIONS) if(NOT MSVC) set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS -Wno-deprecated-declarations) +else() + # The storage regression tests exceed the default COFF section limit. + set_source_files_properties(OfflineStorageTests.cpp PROPERTIES COMPILE_OPTIONS /bigobj) endif() # Enable Azure Monitor unit tests when the module is present. diff --git a/tests/unittests/OfflineStorageTests.cpp b/tests/unittests/OfflineStorageTests.cpp index 9f3f5fc35..50012ee50 100644 --- a/tests/unittests/OfflineStorageTests.cpp +++ b/tests/unittests/OfflineStorageTests.cpp @@ -8,13 +8,20 @@ #include "offline/IOfflineStorageProvider.hpp" #include "offline/StorageObserver.hpp" #include "offline/MemoryStorage.hpp" +#include "pal/TaskDispatcher_CAPI.hpp" #include "NullObjects.hpp" #include +#include +#include #include +#include +#include #include +#include #include #include +#include #include #include @@ -186,6 +193,28 @@ namespace ILogConfiguration& GetLogConfiguration() override { return config; } }; + class CountingLogManager : public ConfigurableLogManager + { + public: + bool StartActivity() override + { + ++activityCount; + ++activeActivities; + return true; + } + + void EndActivity() override { --activeActivities; } + + std::atomic activityCount{0}; + std::atomic activeActivities{0}; + }; + + class PausedLogManager : public ConfigurableLogManager + { + public: + bool StartActivity() override { return false; } + }; + // Remove a SQLite db file along with its WAL-mode companion files // (-wal/-shm/-journal), which would otherwise accumulate in the temp dir. void RemoveDbFiles(const std::string& path) @@ -226,6 +255,89 @@ namespace } std::vector m_tasks; }; + + class ManualTaskDispatcher : public ITaskDispatcher + { + public: + enum class Behavior { Hold, Inline, Drop, ThrowOnce }; + + void Join() override + { + Take(); + } + + void Queue(Task* task) override + { + std::unique_ptr owned(task); + EXPECT_EQ(static_cast(task->Type), static_cast(Task::Call)); + const unsigned call = ++queueCalls; + if (beforeQueue) + { + beforeQueue(); + } + if (behavior == Behavior::Inline) + { + (*owned)(); + return; + } + if (behavior == Behavior::Drop) + { + return; + } +#if HAVE_EXCEPTIONS + if (behavior == Behavior::ThrowOnce && call == 1) + { + throw std::runtime_error("queue failed"); + } +#else + UNREFERENCED_PARAMETER(call); +#endif + std::lock_guard lock(m_mutex); + EXPECT_EQ(m_task, nullptr); + m_task = std::move(owned); + } + + bool Cancel(Task* task, uint64_t = 0) override + { + std::unique_ptr cancelled; + { + std::lock_guard lock(m_mutex); + if (m_task.get() != task) + { + return false; + } + cancelled = std::move(m_task); + } + return true; + } + + std::unique_ptr Take() + { + std::lock_guard lock(m_mutex); + return std::move(m_task); + } + + Behavior behavior = Behavior::Hold; + std::atomic queueCalls{0}; + std::function beforeQueue; + + private: + std::mutex m_mutex; + std::unique_ptr m_task; + }; + + void EVTSDK_LIBABI_CDECL QueueInlineCapiTask(evt_task_t* task, task_callback_fn_t callback) + { + EXPECT_EQ(task->delayMs, 0u); + callback(task->id); + } + + bool EVTSDK_LIBABI_CDECL CancelCapiTask(const char*) + { + return true; + } + + void EVTSDK_LIBABI_CDECL JoinCapiTasks() {} } namespace MAT_NS_BEGIN { @@ -391,6 +503,667 @@ TEST_P(OfflineStorageRetrievalExceptionTests, PartialRetrievalReleasesRecordsToT INSTANTIATE_TEST_SUITE_P(MemoryAndDisk, OfflineStorageRetrievalExceptionTests, Combine(Bool(), Bool())); #endif +class OfflineStorageLifecycleTests : public Test +{ +protected: + void SetUp() override + { + config[CFG_INT_RAM_QUEUE_SIZE] = 1; + config[CFG_BOOL_ENABLE_BATCHED_STORAGE_FLUSH] = true; + memory = std::make_shared(logManager, config); + disk = std::make_shared>(); + auto provider = std::make_shared(memory, disk); + handler.reset(new OfflineStorageHandler(logManager, config, dispatcher, provider)); + EXPECT_CALL(*disk, Initialize(Ref(*handler))); + handler->Initialize(observer); + } + + StorageRecord Record(const char* id) + { + return StorageRecord(id, "tenant-token", EventLatency_Normal, + EventPersistence_Normal, 1, std::vector{'x'}); + } + + bool StoreTwo() + { + return handler->StoreRecord(Record("first")) && handler->StoreRecord(Record("second")); + } + + void ExpectSaved(size_t count) + { + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Invoke([count](std::vector& records) + { + EXPECT_EQ(records.size(), count); + return records.size(); + })); + EXPECT_CALL(observer, OnStorageRecordsSaved(count)); + } + + CountingLogManager logManager; + NiceMock config; + ManualTaskDispatcher dispatcher; + StrictMock observer; + std::shared_ptr memory; + std::shared_ptr> disk; + std::unique_ptr handler; +}; + +TEST_F(OfflineStorageLifecycleTests, ShutdownCancelsQueuedFlushAndPersistsRecords) +{ + ASSERT_TRUE(StoreTwo()); + ExpectSaved(2); + EXPECT_CALL(*disk, Shutdown()); + + std::promise finished; + auto done = finished.get_future(); + std::thread shutdown([&] { + handler->Shutdown(); + finished.set_value(); + }); + const auto status = done.wait_for(std::chrono::seconds(5)); + if (status != std::future_status::ready) + { + auto task = dispatcher.Take(); + if (task) + { + (*task)(); + } + } + shutdown.join(); + EXPECT_EQ(status, std::future_status::ready); + EXPECT_EQ(dispatcher.Take(), nullptr); + EXPECT_FALSE(handler->StoreRecord(Record("after-shutdown"))); + const unsigned activitiesAfterShutdown = logManager.activityCount.load(); + handler->Flush(); + EXPECT_EQ(logManager.activityCount.load(), activitiesAfterShutdown); +} + +TEST_F(OfflineStorageLifecycleTests, ShutdownWaitsForDirectFlushObserver) +{ + ASSERT_TRUE(StoreTwo()); + auto task = dispatcher.Take(); + ASSERT_NE(task, nullptr); + std::promise entered; + auto enteredFuture = entered.get_future(); + std::promise release; + auto releaseFuture = release.get_future().share(); + std::atomic shutdownCalls{0}; + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Invoke([](std::vector& records) { return records.size(); })); + EXPECT_CALL(observer, OnStorageRecordsSaved(2)) + .WillOnce(Invoke([&](size_t) { + entered.set_value(); + releaseFuture.wait(); + })); + EXPECT_CALL(*disk, Shutdown()).WillOnce(Invoke([&] { ++shutdownCalls; })); + + std::thread flush([&] { handler->Flush(); }); + if (enteredFuture.wait_for(std::chrono::seconds(5)) != std::future_status::ready) + { + release.set_value(); + flush.join(); + FAIL() << "Direct flush did not notify its observer"; + } + std::promise started; + auto startedFuture = started.get_future(); + std::promise finished; + auto done = finished.get_future(); + std::thread shutdown([&] { + started.set_value(); + handler->Shutdown(); + finished.set_value(); + }); + startedFuture.wait(); + EXPECT_EQ(done.wait_for(std::chrono::milliseconds(200)), std::future_status::timeout); + EXPECT_EQ(shutdownCalls.load(), 0u); + release.set_value(); + flush.join(); + shutdown.join(); + EXPECT_EQ(done.wait_for(std::chrono::seconds(0)), std::future_status::ready); + const unsigned activitiesAfterShutdown = logManager.activityCount.load(); + (*task)(); + EXPECT_EQ(logManager.activityCount.load(), activitiesAfterShutdown); +} + +TEST_F(OfflineStorageLifecycleTests, ConcurrentShutdownRunsStorageShutdownOnce) +{ + std::promise entered; + auto enteredFuture = entered.get_future(); + std::promise release; + auto releaseFuture = release.get_future().share(); + EXPECT_CALL(*disk, Shutdown()).WillOnce(Invoke([&] { + entered.set_value(); + releaseFuture.wait(); + })); + + std::thread first([&] { handler->Shutdown(); }); + if (enteredFuture.wait_for(std::chrono::seconds(5)) != std::future_status::ready) + { + release.set_value(); + first.join(); + FAIL() << "First shutdown did not reach storage"; + } + std::promise started; + auto startedFuture = started.get_future(); + std::promise finished; + auto done = finished.get_future(); + std::thread second([&] { + started.set_value(); + handler->Shutdown(); + finished.set_value(); + }); + startedFuture.wait(); + EXPECT_EQ(done.wait_for(std::chrono::milliseconds(200)), std::future_status::timeout); + release.set_value(); + first.join(); + second.join(); + EXPECT_EQ(done.wait_for(std::chrono::seconds(0)), std::future_status::ready); + handler->Shutdown(); +} + +TEST_F(OfflineStorageLifecycleTests, ShutdownWaitsForRunningFlush) +{ + ASSERT_TRUE(StoreTwo()); + auto task = dispatcher.Take(); + ASSERT_NE(task, nullptr); + std::promise entered; + auto enteredFuture = entered.get_future(); + std::promise release; + auto releaseFuture = release.get_future().share(); + std::atomic batchCalls{0}; + std::atomic directlyStored{0}; + EXPECT_CALL(*disk, StoreRecords(_)) + .WillRepeatedly(Invoke([&](std::vector& records) { + if (++batchCalls == 1) + { + entered.set_value(); + releaseFuture.wait(); + } + return records.size(); + })); + EXPECT_CALL(*disk, StoreRecord(_)) + .WillRepeatedly(Invoke([&](StorageRecord const&) { + ++directlyStored; + return true; + })); + EXPECT_CALL(observer, OnStorageRecordsSaved(_)).Times(AtLeast(1)); + EXPECT_CALL(*disk, Shutdown()); + + std::thread worker([task = std::move(task)] { (*task)(); }); + if (enteredFuture.wait_for(std::chrono::seconds(5)) != std::future_status::ready) + { + release.set_value(); + worker.join(); + FAIL() << "Scheduled flush did not enter storage"; + } + std::promise started; + auto startedFuture = started.get_future(); + std::promise finished; + auto done = finished.get_future(); + std::thread shutdown([&] { + started.set_value(); + handler->Shutdown(); + finished.set_value(); + }); + startedFuture.wait(); + for (unsigned i = 0; i < 500 && directlyStored.load() == 0; ++i) + { + auto id = "probe-" + std::to_string(i); + handler->StoreRecord(Record(id.c_str())); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + EXPECT_EQ(directlyStored.load(), 1u); + EXPECT_EQ(done.wait_for(std::chrono::seconds(0)), std::future_status::timeout); + release.set_value(); + worker.join(); + shutdown.join(); + EXPECT_EQ(done.wait_for(std::chrono::seconds(0)), std::future_status::ready); +} + +TEST_F(OfflineStorageLifecycleTests, StoresDuringDrainReachDiskBeforeShutdown) +{ + ASSERT_TRUE(handler->StoreRecord(Record("first"))); + std::promise entered; + auto enteredFuture = entered.get_future(); + std::promise release; + auto releaseFuture = release.get_future().share(); + dispatcher.beforeQueue = [&] { + entered.set_value(); + releaseFuture.wait(); + }; + std::atomic directlyStored{0}; + size_t batched = 0; + EXPECT_CALL(*disk, StoreRecord(_)) + .WillRepeatedly(Invoke([&](StorageRecord const&) { + ++directlyStored; + return true; + })); + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Invoke([&](std::vector& records) { + batched = records.size(); + return records.size(); + })); + EXPECT_CALL(observer, OnStorageRecordsSaved(_)); + EXPECT_CALL(*disk, Shutdown()); + + std::thread producer([&] { EXPECT_TRUE(handler->StoreRecord(Record("second"))); }); + if (enteredFuture.wait_for(std::chrono::seconds(5)) != std::future_status::ready) + { + release.set_value(); + producer.join(); + FAIL() << "Store did not reach task submission"; + } + std::promise started; + auto startedFuture = started.get_future(); + std::promise finished; + auto done = finished.get_future(); + std::thread shutdown([&] { + started.set_value(); + handler->Shutdown(); + finished.set_value(); + }); + startedFuture.wait(); + size_t accepted = 2; + for (unsigned i = 0; i < 500 && directlyStored.load() == 0; ++i) + { + auto id = "during-drain-" + std::to_string(i); + if (handler->StoreRecord(Record(id.c_str()))) + { + ++accepted; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + EXPECT_EQ(directlyStored.load(), 1u); + EXPECT_EQ(done.wait_for(std::chrono::seconds(0)), std::future_status::timeout); + release.set_value(); + producer.join(); + shutdown.join(); + EXPECT_EQ(batched + directlyStored.load(), accepted); + EXPECT_EQ(dispatcher.Take(), nullptr); +} + +TEST_F(OfflineStorageLifecycleTests, DroppedTaskAllowsAnotherFlushToBeScheduled) +{ + dispatcher.behavior = ManualTaskDispatcher::Behavior::Drop; + ASSERT_TRUE(StoreTwo()); + ASSERT_TRUE(handler->StoreRecord(Record("third"))); + EXPECT_EQ(dispatcher.queueCalls.load(), 2u); + ExpectSaved(3); + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); +} + +TEST_F(OfflineStorageLifecycleTests, InlineDispatcherRunsFlushWithoutDeadlock) +{ + dispatcher.behavior = ManualTaskDispatcher::Behavior::Inline; + ExpectSaved(2); + ASSERT_TRUE(StoreTwo()); + EXPECT_EQ(dispatcher.queueCalls.load(), 1u); + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); +} + +#if HAVE_EXCEPTIONS +TEST_F(OfflineStorageLifecycleTests, FailedQueueDoesNotBlockLaterScheduling) +{ + dispatcher.behavior = ManualTaskDispatcher::Behavior::ThrowOnce; + ASSERT_TRUE(handler->StoreRecord(Record("first"))); + EXPECT_THROW(handler->StoreRecord(Record("second")), std::runtime_error); + ASSERT_TRUE(handler->StoreRecord(Record("third"))); + EXPECT_EQ(dispatcher.queueCalls.load(), 2u); + ExpectSaved(3); + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); +} + +TEST_F(OfflineStorageLifecycleTests, FailedDirectFlushReleasesActivityAndCanBeRetried) +{ + ASSERT_TRUE(StoreTwo()); + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Invoke([](std::vector& records) -> size_t { + records.clear(); + throw std::runtime_error("disk write failed"); + })) + .WillOnce(Invoke([](std::vector& records) { return records.size(); })); + EXPECT_CALL(observer, OnStorageRecordsSaved(2)); + EXPECT_THROW(handler->Flush(), std::runtime_error); + EXPECT_EQ(logManager.activeActivities.load(), 0); + EXPECT_EQ(memory->GetRecordCount(), 2u); + handler->Flush(); + EXPECT_EQ(logManager.activeActivities.load(), 0); + EXPECT_EQ(memory->GetRecordCount(), 0u); + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); +} + +TEST_F(OfflineStorageLifecycleTests, ConcurrentShutdownRetriesFailedAttempt) +{ + std::promise entered; + auto enteredFuture = entered.get_future(); + std::promise release; + auto releaseFuture = release.get_future().share(); + EXPECT_CALL(*disk, Shutdown()) + .WillOnce(Invoke([&] { + entered.set_value(); + releaseFuture.wait(); + throw std::runtime_error("shutdown failed"); + })) + .WillOnce(Return()); + + std::thread first([&] { EXPECT_THROW(handler->Shutdown(), std::runtime_error); }); + if (enteredFuture.wait_for(std::chrono::seconds(5)) != std::future_status::ready) + { + release.set_value(); + first.join(); + FAIL() << "First shutdown did not reach storage"; + } + std::promise started; + auto startedFuture = started.get_future(); + std::promise finished; + auto done = finished.get_future(); + std::thread second([&] { + started.set_value(); + EXPECT_NO_THROW(handler->Shutdown()); + finished.set_value(); + }); + startedFuture.wait(); + EXPECT_EQ(done.wait_for(std::chrono::milliseconds(200)), std::future_status::timeout); + release.set_value(); + first.join(); + second.join(); + EXPECT_EQ(done.wait_for(std::chrono::seconds(0)), std::future_status::ready); + handler->Shutdown(); +} + +TEST_F(OfflineStorageLifecycleTests, FailedShutdownFlushCanBeRetried) +{ + ASSERT_TRUE(StoreTwo()); + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Throw(std::runtime_error("disk write failed"))) + .WillOnce(Invoke([](std::vector& records) { return records.size(); })); + EXPECT_CALL(observer, OnStorageRecordsSaved(2)); + EXPECT_CALL(*disk, Shutdown()); + EXPECT_THROW(handler->Shutdown(), std::runtime_error); + EXPECT_EQ(memory->GetRecordCount(), 2u); + handler->Shutdown(); + EXPECT_EQ(memory->GetRecordCount(), 0u); +} + +TEST_F(OfflineStorageLifecycleTests, ReinitializeRetriesFailedShutdownBeforeReplacingStorage) +{ + ASSERT_TRUE(StoreTwo()); + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Throw(std::runtime_error("disk write failed"))) + .WillOnce(Invoke([](std::vector& records) { return records.size(); })); + EXPECT_CALL(observer, OnStorageRecordsSaved(2)); + EXPECT_CALL(*disk, Shutdown()).Times(2); + EXPECT_THROW(handler->Shutdown(), std::runtime_error); + EXPECT_CALL(*disk, Initialize(Ref(*handler))); + handler->Initialize(observer); + EXPECT_EQ(memory->GetRecordCount(), 0u); + handler->Shutdown(); +} + +TEST(OfflineStorageHandlerFlushTests, RetryingDiskShutdownDoesNotRepeatMemoryShutdown) +{ + ConfigurableLogManager logManager; + NiceMock config; + ManualTaskDispatcher dispatcher; + StrictMock observer; + config[CFG_INT_RAM_QUEUE_SIZE] = 1024 * 1024; + config[CFG_BOOL_ENABLE_BATCHED_STORAGE_FLUSH] = true; + auto memory = std::make_shared>(); + auto disk = std::make_shared>(); + auto provider = std::make_shared(memory, disk); + OfflineStorageHandler handler(logManager, config, dispatcher, provider); + EXPECT_CALL(*disk, Initialize(Ref(handler))); + EXPECT_CALL(*memory, Initialize(Ref(handler))); + handler.Initialize(observer); + EXPECT_CALL(*memory, GetSize()).WillOnce(Return(0)); + EXPECT_CALL(*memory, Shutdown()).Times(1); + EXPECT_CALL(*disk, Shutdown()) + .WillOnce(Throw(std::runtime_error("shutdown failed"))) + .WillOnce(Return()); + EXPECT_THROW(handler.Shutdown(), std::runtime_error); + handler.Shutdown(); +} +#endif + +TEST_F(OfflineStorageLifecycleTests, PartialBatchMutationRetainsRecordsForRetry) +{ + ASSERT_TRUE(StoreTwo()); + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Invoke([](std::vector& records) -> size_t { + records.clear(); + return 1; + })) + .WillOnce(Invoke([](std::vector& records) { + EXPECT_EQ(records.size(), 2u); + return records.size(); + })); + EXPECT_CALL(observer, OnStorageRecordsSaved(1)); + EXPECT_CALL(observer, OnStorageRecordsSaved(2)); + handler->Flush(); + EXPECT_EQ(memory->GetRecordCount(), 2u); + handler->Flush(); + EXPECT_EQ(memory->GetRecordCount(), 0u); + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); +} + +TEST_F(OfflineStorageLifecycleTests, SavedObserverCanWaitForAnotherFlush) +{ + ASSERT_TRUE(StoreTwo()); + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Invoke([](std::vector& records) { return records.size(); })); + std::promise finished; + auto done = finished.get_future(); + std::thread nested; + EXPECT_CALL(observer, OnStorageRecordsSaved(2)) + .WillOnce(Invoke([&](size_t) { + nested = std::thread([&] { + handler->Flush(); + finished.set_value(); + }); + EXPECT_EQ(done.wait_for(std::chrono::seconds(5)), std::future_status::ready); + })); + handler->Flush(); + if (nested.joinable()) + { + nested.join(); + } + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); +} + +TEST_F(OfflineStorageLifecycleTests, ReinitializeWithoutMemoryCacheStoresDirectlyToDisk) +{ + config[CFG_INT_RAM_QUEUE_SIZE] = 0; + EXPECT_CALL(*disk, Shutdown()); + EXPECT_CALL(*disk, Initialize(Ref(*handler))); + handler->Initialize(observer); + + EXPECT_CALL(*disk, StoreRecord(_)).WillOnce(Return(true)); + EXPECT_TRUE(handler->StoreRecord(Record("direct"))); + EXPECT_EQ(memory->GetRecordCount(), 0u); + + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); +} + +TEST_F(OfflineStorageLifecycleTests, OldDequeuedTaskStaysInertAfterReinitialize) +{ + ASSERT_TRUE(StoreTwo()); + auto oldTask = dispatcher.Take(); + ASSERT_NE(oldTask, nullptr); + ExpectSaved(2); + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); + + EXPECT_CALL(*disk, Initialize(Ref(*handler))); + handler->Initialize(observer); + ASSERT_TRUE(StoreTwo()); + const unsigned activitiesBeforeTask = logManager.activityCount.load(); + (*oldTask)(); + oldTask.reset(); + EXPECT_EQ(logManager.activityCount.load(), activitiesBeforeTask); + auto currentTask = dispatcher.Take(); + ASSERT_NE(currentTask, nullptr); + ExpectSaved(2); + (*currentTask)(); + EXPECT_GT(logManager.activityCount.load(), activitiesBeforeTask); + EXPECT_EQ(memory->GetRecordCount(), 0u); + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); +} + +TEST_F(OfflineStorageLifecycleTests, ReinitializeWhileAcceptingClosesOldGeneration) +{ + ASSERT_TRUE(StoreTwo()); + auto oldTask = dispatcher.Take(); + ASSERT_NE(oldTask, nullptr); + ExpectSaved(2); + EXPECT_CALL(*disk, Shutdown()); + EXPECT_CALL(*disk, Initialize(Ref(*handler))); + handler->Initialize(observer); + + ASSERT_TRUE(StoreTwo()); + const unsigned activitiesBeforeTask = logManager.activityCount.load(); + (*oldTask)(); + oldTask.reset(); + EXPECT_EQ(logManager.activityCount.load(), activitiesBeforeTask); + auto currentTask = dispatcher.Take(); + ASSERT_NE(currentTask, nullptr); + ExpectSaved(2); + (*currentTask)(); + EXPECT_EQ(memory->GetRecordCount(), 0u); + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); +} + +TEST_F(OfflineStorageLifecycleTests, OldCompletionDoesNotClearNewerQueuedFlush) +{ + ASSERT_TRUE(StoreTwo()); + auto oldTask = dispatcher.Take(); + ASSERT_NE(oldTask, nullptr); + std::promise entered; + auto enteredFuture = entered.get_future(); + std::promise release; + auto releaseFuture = release.get_future().share(); + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Invoke([&](std::vector& records) { + entered.set_value(); + releaseFuture.wait(); + return records.size(); + })) + .WillOnce(Invoke([](std::vector& records) { return records.size(); })); + EXPECT_CALL(observer, OnStorageRecordsSaved(2)).Times(2); + std::thread worker([task = std::move(oldTask)] { (*task)(); }); + if (enteredFuture.wait_for(std::chrono::seconds(5)) != std::future_status::ready) + { + release.set_value(); + worker.join(); + FAIL() << "Scheduled flush did not enter storage"; + } + EXPECT_TRUE(handler->StoreRecord(Record("third"))); + EXPECT_TRUE(handler->StoreRecord(Record("fourth"))); + release.set_value(); + worker.join(); + auto newTask = dispatcher.Take(); + ASSERT_NE(newTask, nullptr); + (*newTask)(); + EXPECT_EQ(memory->GetRecordCount(), 0u); + EXPECT_CALL(*disk, Shutdown()); + handler->Shutdown(); +} + +TEST(OfflineStorageHandlerFlushTests, ShutdownFlushesWhenActivitiesArePaused) +{ + PausedLogManager logManager; + NiceMock config; + ManualTaskDispatcher dispatcher; + StrictMock observer; + config[CFG_INT_RAM_QUEUE_SIZE] = 1024 * 1024; + config[CFG_BOOL_ENABLE_BATCHED_STORAGE_FLUSH] = true; + auto memory = std::make_shared(logManager, config); + auto disk = std::make_shared>(); + auto provider = std::make_shared(memory, disk); + OfflineStorageHandler handler(logManager, config, dispatcher, provider); + EXPECT_CALL(*disk, Initialize(Ref(handler))); + handler.Initialize(observer); + StorageRecord record("event", "tenant-token", EventLatency_Normal, + EventPersistence_Normal, 1, std::vector{'x'}); + ASSERT_TRUE(handler.StoreRecord(record)); + handler.Flush(); + EXPECT_CALL(*disk, StoreRecords(_)).WillOnce(Return(1)); + EXPECT_CALL(observer, OnStorageRecordsSaved(1)); + EXPECT_CALL(*disk, Shutdown()); + handler.Shutdown(); +} + +TEST(OfflineStorageHandlerFlushTests, CapiDispatcherReceivesImmediateFlush) +{ + ConfigurableLogManager logManager; + NiceMock config; + PAL::TaskDispatcher_CAPI dispatcher(&QueueInlineCapiTask, &CancelCapiTask, &JoinCapiTasks); + StrictMock observer; + config[CFG_INT_RAM_QUEUE_SIZE] = 1; + config[CFG_BOOL_ENABLE_BATCHED_STORAGE_FLUSH] = true; + auto memory = std::make_shared(logManager, config); + auto disk = std::make_shared>(); + auto provider = std::make_shared(memory, disk); + OfflineStorageHandler handler(logManager, config, dispatcher, provider); + EXPECT_CALL(*disk, Initialize(Ref(handler))); + handler.Initialize(observer); + EXPECT_CALL(*disk, StoreRecords(_)).WillOnce(Return(2)); + EXPECT_CALL(observer, OnStorageRecordsSaved(2)); + StorageRecord first("first", "tenant-token", EventLatency_Normal, + EventPersistence_Normal, 1, std::vector{'x'}); + StorageRecord second("second", "tenant-token", EventLatency_Normal, + EventPersistence_Normal, 1, std::vector{'y'}); + ASSERT_TRUE(handler.StoreRecord(first)); + ASSERT_TRUE(handler.StoreRecord(second)); + EXPECT_CALL(*disk, Shutdown()); + handler.Shutdown(); +} + +TEST(OfflineStorageHandlerFlushTests, DequeuedFlushCannotRunAfterShutdown) +{ + CountingLogManager logManager; + NiceMock config; + ManualTaskDispatcher dispatcher; + StrictMock observer; + config[CFG_INT_RAM_QUEUE_SIZE] = 1; + config[CFG_BOOL_ENABLE_BATCHED_STORAGE_FLUSH] = true; + auto memory = std::make_shared(logManager, config); + auto disk = std::make_shared>(); + auto provider = std::make_shared(memory, disk); + OfflineStorageHandler handler(logManager, config, dispatcher, provider); + EXPECT_CALL(*disk, Initialize(Ref(handler))); + handler.Initialize(observer); + + StorageRecord first("first", "tenant-token", EventLatency_Normal, + EventPersistence_Normal, 1, std::vector{'x'}); + StorageRecord second("second", "tenant-token", EventLatency_Normal, + EventPersistence_Normal, 1, std::vector{'y'}); + ASSERT_TRUE(handler.StoreRecord(first)); + ASSERT_TRUE(handler.StoreRecord(second)); + auto task = dispatcher.Take(); + ASSERT_NE(task, nullptr); + + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Invoke([](std::vector& records) { return records.size(); })); + EXPECT_CALL(observer, OnStorageRecordsSaved(2)); + handler.Flush(); + + EXPECT_CALL(*disk, Shutdown()); + handler.Shutdown(); + const unsigned activitiesBeforeTask = logManager.activityCount.load(); + (*task)(); + EXPECT_EQ(logManager.activityCount.load(), activitiesBeforeTask); +} + TEST(OfflineStorageHandlerFlushTests, FailedMemoryRequeueIsReportedAndDropped) { NullLogManager logManager; @@ -512,7 +1285,12 @@ TEST(OfflineStorageHandlerFlushTests, BatchedFlushKeepsMemoryOnlyRecordsOffDisk) EXPECT_CALL(*memory, StoreRecord(Field(&StorageRecord::id, "memory-only"))) .WillOnce(Return(true)); EXPECT_CALL(*disk, StoreRecord(_)).Times(0); - EXPECT_CALL(*disk, StoreRecords(_)).WillOnce(Return(1)); + EXPECT_CALL(*disk, StoreRecords(_)) + .WillOnce(Invoke([](std::vector& batch) -> size_t { + EXPECT_THAT(batch, ElementsAre(Field(&StorageRecord::id, "persisted"))); + batch.clear(); + return 1; + })); EXPECT_CALL(observer, OnStorageRecordsSaved(1)); handler.Flush(); diff --git a/tests/unittests/PalTests.cpp b/tests/unittests/PalTests.cpp index 9d4ba3891..cf37c8c56 100644 --- a/tests/unittests/PalTests.cpp +++ b/tests/unittests/PalTests.cpp @@ -592,6 +592,25 @@ namespace bool m_successorRan = false; }; + class TrackedImmediateTask final : public MAT::Task + { + public: + TrackedImmediateTask(std::atomic& ran, std::atomic& destroyed) : + m_ran(ran), m_destroyed(destroyed) + { + Type = MAT::Task::Call; + TargetTime = 0; + TypeName = "TrackedImmediateTask"; + } + + ~TrackedImmediateTask() noexcept override { m_destroyed.store(true); } + void operator()() override { m_ran.store(true); } + + private: + std::atomic& m_ran; + std::atomic& m_destroyed; + }; + class SelfDisposeHelper { public: @@ -682,6 +701,33 @@ TEST_F(PalTests, ScheduleTaskAfterWorkerThreadJoinReturnsNoOpHandle) EXPECT_TRUE(handle.Cancel()); } +TEST_F(PalTests, CancelRemovesQueuedImmediateTask) +{ + auto dispatcher = PAL::WorkerThreadFactory::Create(); + BlockingCancellationTarget blocker; + auto running = PAL::scheduleTask( + dispatcher.get(), 0, &blocker, &BlockingCancellationTarget::Block); + (void)running; + if (!blocker.WaitUntilEntered()) + { + blocker.Release(); + dispatcher->Join(); + FAIL() << "Worker did not start the blocking task"; + } + + std::atomic ran{false}; + std::atomic destroyed{false}; + auto task = std::make_unique(ran, destroyed); + auto* identity = task.get(); + dispatcher->Queue(task.release()); + EXPECT_TRUE(dispatcher->Cancel(identity)); + EXPECT_TRUE(destroyed.load()); + blocker.Release(); + dispatcher->Join(); + EXPECT_FALSE(ran.load()); + EXPECT_TRUE(destroyed.load()); +} + TEST_F(PalTests, ScheduleTaskHandleClearsAfterWorkerThreadCallbackCompletes) { auto dispatcher = PAL::WorkerThreadFactory::Create();