diff --git a/Makefile b/Makefile index 0ac17aed..2c06afe5 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,11 @@ CFLAGS += -DSPLINTERDB_PLATFORM_DIR=$(PLATFORM_DIR) GIT_VERSION := "$(shell git describe --abbrev=8 --dirty --always --tags)" GIT_VERSION_CFLAGS += -DGIT_VERSION=\"$(GIT_VERSION)\" +# Fix annoying warning about avx-256 vs avx-512 +ifeq "$(findstring clang, $(CC))" "clang" + CFLAGS += -Wno-invalid-feature-combination +endif + cpu_arch := $(shell uname -p) ifeq ($(cpu_arch),x86_64) # not supported on ARM64 @@ -427,11 +432,12 @@ PLATFORM_IO_SYS = $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/platform_io.o \ UTIL_SYS = $(OBJDIR)/$(SRCDIR)/util.o $(PLATFORM_SYS) -CLOCKCACHE_SYS = $(OBJDIR)/$(SRCDIR)/clockcache.o \ - $(OBJDIR)/$(SRCDIR)/allocator.o \ - $(OBJDIR)/$(SRCDIR)/rc_allocator.o \ - $(OBJDIR)/$(SRCDIR)/task.o \ - $(UTIL_SYS) \ +CLOCKCACHE_SYS = $(OBJDIR)/$(SRCDIR)/clockcache.o \ + $(OBJDIR)/$(SRCDIR)/allocator.o \ + $(OBJDIR)/$(SRCDIR)/rc_allocator.o \ + $(OBJDIR)/$(SRCDIR)/task.o \ + $(OBJDIR)/$(SRCDIR)/writeback_set.o \ + $(UTIL_SYS) \ $(PLATFORM_IO_SYS) BTREE_SYS = $(OBJDIR)/$(SRCDIR)/btree.o \ @@ -468,6 +474,14 @@ $(BINDIR)/$(UNITDIR)/btree_stress_test: $(OBJDIR)/$(UNIT_TESTSDIR)/btree_test_co $(COMMON_UNIT_TESTOBJ) \ $(BTREE_SYS) +# Uses btree_test_common only for its init_*_config_from_master_config() +# helpers, which is why it pulls BTREE_SYS rather than just CLOCKCACHE_SYS. +$(BINDIR)/$(UNITDIR)/writeback_set_test: $(OBJDIR)/$(UNIT_TESTSDIR)/btree_test_common.o \ + $(OBJDIR)/$(TESTS_DIR)/config.o \ + $(OBJDIR)/$(TESTS_DIR)/test_data.o \ + $(COMMON_UNIT_TESTOBJ) \ + $(BTREE_SYS) + $(BINDIR)/$(UNITDIR)/splinter_test: $(COMMON_TESTOBJ) \ $(COMMON_UNIT_TESTOBJ) \ $(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \ @@ -548,6 +562,7 @@ unit/misc_test: $(BINDIR)/$(UNITDIR)/misc_test unit/platform_threads_test: $(BINDIR)/$(UNITDIR)/platform_threads_test unit/btree_test: $(BINDIR)/$(UNITDIR)/btree_test unit/btree_stress_test: $(BINDIR)/$(UNITDIR)/btree_stress_test +unit/writeback_set_test: $(BINDIR)/$(UNITDIR)/writeback_set_test unit/splinter_test: $(BINDIR)/$(UNITDIR)/splinter_test unit/splinterdb_quick_test: $(BINDIR)/$(UNITDIR)/splinterdb_quick_test unit/splinterdb_stress_test: $(BINDIR)/$(UNITDIR)/splinterdb_stress_test diff --git a/docs/limitations.md b/docs/limitations.md index 884c5e0f..d500a73d 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -9,7 +9,6 @@ Thus, SplinterDB is provided *as-is* given the following limitations and missing * SplinterDB on-disk format is not versioned (Data may not survive software upgrades.) * Single 4KiB page size, with fixed extent size of 32 pages/extent. * Key and value size need to be less than the page size. -* SplinterDB does not expose an API to force the latest write to be durable (e.g., fsync/commit.) * SplinterDB disk size cannot be changed once configured. * SplinterDB does not have a public API for the experimental async features. * SplinterDB does not retain configuration parameters and metadata. (These cannot diff --git a/docs/site/content/docs/limitations.md b/docs/site/content/docs/limitations.md index eb15e801..a170ee6f 100644 --- a/docs/site/content/docs/limitations.md +++ b/docs/site/content/docs/limitations.md @@ -12,7 +12,6 @@ Thus, SplinterDB is provided *as-is* given the following limitations and missing between 8 to 105 bytes. Support for smaller key-sizes is experimental. * The application must specify the minimum and maximum of the key range. * SplinterDB on-disk size is fixed at compile time. -* SplinterDB does not expose an API to force the latest write to be durable (e.g., fsync/commit.) * SplinterDB disk size cannot be changed once configured. * SplinterDB does not have a public API for the experimental async features. * SplinterDB does not retain configuration parameters and metadata. (These cannot diff --git a/examples/splinterdb_custom_ipv4_addr_sortcmp_example.c b/examples/splinterdb_custom_ipv4_addr_sortcmp_example.c index 91d6c59f..7a251f1c 100644 --- a/examples/splinterdb_custom_ipv4_addr_sortcmp_example.c +++ b/examples/splinterdb_custom_ipv4_addr_sortcmp_example.c @@ -172,7 +172,7 @@ main() start_key = "100.101.102.103"; do_iterate_from(spl_handle, start_key); - splinterdb_close(&spl_handle); + splinterdb_close(&spl_handle, FALSE); printf("Shutdown SplinterDB instance, dbname '%s'.\n\n", DB_FILE_NAME); return rc; diff --git a/examples/splinterdb_intro_example.c b/examples/splinterdb_intro_example.c index e646f110..7c47575e 100644 --- a/examples/splinterdb_intro_example.c +++ b/examples/splinterdb_intro_example.c @@ -94,7 +94,13 @@ main() printf("\n"); printf("Shutdown and reopen SplinterDB instance ...\n"); - splinterdb_close(&spl_handle); + rc = splinterdb_close(&spl_handle, FALSE); + if (rc) { + printf("Error shutting down SplinterDB instance, dbname '%s' (rc=%d).\n", + DB_FILE_NAME, + rc); + return rc; + } rc = splinterdb_open(&splinterdb_cfg, &spl_handle); if (rc) { @@ -129,7 +135,7 @@ main() printf("Found %d key-value pairs\n\n", i); - splinterdb_close(&spl_handle); + splinterdb_close(&spl_handle, FALSE); printf("Shutdown SplinterDB instance, dbname '%s'.\n\n", DB_FILE_NAME); return rc; diff --git a/examples/splinterdb_iterators_example.c b/examples/splinterdb_iterators_example.c index 35c6d553..8adb2c25 100644 --- a/examples/splinterdb_iterators_example.c +++ b/examples/splinterdb_iterators_example.c @@ -90,7 +90,7 @@ main() start_key = "www.twitter.com"; do_iterate_from(spl_handle, start_key); - splinterdb_close(&spl_handle); + splinterdb_close(&spl_handle, FALSE); printf("Shutdown SplinterDB instance, dbname '%s'.\n\n", DB_FILE_NAME); return rc; diff --git a/examples/splinterdb_optimize_example.c b/examples/splinterdb_optimize_example.c index aadfab77..1d0ddfa6 100644 --- a/examples/splinterdb_optimize_example.c +++ b/examples/splinterdb_optimize_example.c @@ -109,7 +109,7 @@ main(int argc, char **argv) fprintf(stderr, "splinterdb_optimize failed: %d\n", rc); } - splinterdb_close(&spl); + splinterdb_close(&spl, FALSE); return rc; } diff --git a/examples/splinterdb_wide_values_example.c b/examples/splinterdb_wide_values_example.c index 43497fd4..660e2119 100644 --- a/examples/splinterdb_wide_values_example.c +++ b/examples/splinterdb_wide_values_example.c @@ -111,7 +111,7 @@ main() } splinterdb_lookup_result_deinit(&result); - splinterdb_close(&spl_handle); + splinterdb_close(&spl_handle, FALSE); printf("Shutdown SplinterDB instance, dbname '%s'.\n\n", DB_FILE_NAME); return rc; diff --git a/include/splinterdb/splinterdb.h b/include/splinterdb/splinterdb.h index d9be5a08..326feca6 100644 --- a/include/splinterdb/splinterdb.h +++ b/include/splinterdb/splinterdb.h @@ -102,9 +102,10 @@ typedef struct splinterdb_config { _Bool use_log; // Automatic checkpoints: once the write-ahead log has grown by this many - // bytes, SplinterDB takes a checkpoint, which folds the logged updates into - // the durable tree and reclaims that log's space. This bounds both how much - // log a crash has to replay and how much space the log occupies. + // bytes, SplinterDB arms a checkpoint. The next natural memtable rotation + // cuts the log, after which the checkpoint folds the logged updates into the + // durable tree and reclaims that log's space. This bounds both how much log + // a crash has to replay and how much space the log occupies. // // The trigger is sized in log bytes rather than in updates because the two // are independent: a workload that repeatedly overwrites the same keys grows @@ -117,6 +118,13 @@ typedef struct splinterdb_config { // very large (UINT64_MAX). uint64 checkpoint_log_size_bytes; + // Once an automatic checkpoint is armed, allow the live log to grow by this + // many additional bytes while waiting for a natural memtable rotation. If + // the memtable has not rotated by then, SplinterDB forces a rotation. Zero + // selects a default of twice the memtable capacity; UINT64_MAX effectively + // disables forced rotation while retaining the soft checkpoint trigger. + uint64 checkpoint_log_grace_bytes; + // splinter uint64 memtable_capacity; uint64 fanout; @@ -205,9 +213,30 @@ splinterdb_open(const splinterdb_config *cfg, splinterdb **kvs); // Close a splinterdb // -// This will flush all data to disk and release all resources -void -splinterdb_close(splinterdb **kvs); +// A completed close makes all acknowledged data recoverable and releases all +// resources. +// +// STATUS_OK means all acknowledged data is recoverable. Recovery may still +// need to replay a durable log or rebuild allocator state; that is not a close +// failure. +// +// Without force, any error means shutdown was refused before destructive +// teardown. The database remains open and *kvs is unchanged, so the caller can +// retry or investigate. +// +// With force, teardown always completes. An error means data preservation +// could not be guaranteed; it does not prove that data was actually lost. +// +// force | return code | database closed? | meaning +// --------------------------------------------------------------- +// FALSE | STATUS_OK | YES | acknowledged data is recoverable +// FALSE | any error | NO | shutdown was refused +// TRUE | STATUS_OK | YES | acknowledged data is recoverable +// TRUE | any error | YES | preservation cannot be guaranteed +// +// After STATUS_OK or any forced close, *kvs is freed and set to NULL. +int +splinterdb_close(splinterdb **kvs, bool32 force); //////////////////////////////////// @@ -333,6 +362,9 @@ splinterdb_lookup(splinterdb *kvs, // IN // Updates ///////////////////////////////// +// A successful update is visible to subsequent operations, but is not +// necessarily durable. Use splinterdb_durable_barrier() to establish crash +// durability for a prefix of updates. // Insert a key and value. Overwrites any previous value associated with the // key. @@ -372,6 +404,29 @@ splinterdb_optimize(splinterdb *kvs, _Bool full_leaf_compactions, splinterdb_notification *notification); +///////////////////////////////// +// Durability +///////////////////////////////// + +// Establish a durability barrier. +// +// On success, every update to kvs that linearized before this call began is +// recoverable after a crash or power loss. This includes every successful +// insert, update, or delete that returned before the call began. Updates may +// proceed concurrently; an update overlapping the call may or may not be +// covered. +// +// This establishes durability only. With the write-ahead log enabled, it does +// not promise a checkpoint, log reclamation, a clean cache, or recovery without +// log replay. Without the write-ahead log, SplinterDB obtains the same +// guarantee by checkpointing the tree, which may be substantially more +// expensive. +// +// Returns 0 on success. A nonzero return means the guarantee was not +// established; some or all updates may nevertheless already be durable. +int +splinterdb_durable_barrier(splinterdb *kvs); + /* Iterator API (range query) diff --git a/src/allocator.h b/src/allocator.h index 94b7299b..f1cbb205 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -235,6 +235,19 @@ allocator_get_refcount(allocator *al, uint64 addr) return al->ops->get_ref(al, addr); } +/* + * Discard whatever the refcount map holds and start a rebuild: afterwards only + * the reserved extents are referenced, and + * allocator_recovery_record_reference() supplies the rest until + * allocator_recovery_finish() declares the map usable. + * + * Repeatable, deliberately. Crash recovery rebuilds twice: once counting the + * logs, so that replay is never handed an extent a record it has not reached + * yet depends on, and again from the durable root alone once replay has + * finished and been folded in. The second rebuild is what frees the logs -- + * their extents are simply absent from it -- which is why nothing has to + * enumerate them a second time in order to release them. + */ static inline platform_status allocator_recovery_begin(allocator *al) { diff --git a/src/blob.c b/src/blob.c index f8c0c375..312f4c09 100644 --- a/src/blob.c +++ b/src/blob.c @@ -7,6 +7,22 @@ #define MIN_LIVE_PERCENTAGE (90ULL) +static platform_status +blob_get_descriptor(slice sblob, const blob **blobby) +{ + if (slice_length(sblob) < sizeof(blob)) { + return STATUS_INVALID_STATE; + } + + const blob *candidate = slice_data(sblob); + if (candidate->format != BLOB_FORMAT) { + return STATUS_INVALID_STATE; + } + + *blobby = candidate; + return STATUS_OK; +} + /* If the data is large enough (or close enough to a whole number of * rounded_size pieces), then we just put it entirely into * rounded_size pieces, since this won't waste too much space. @@ -29,6 +45,7 @@ parse_blob(uint64 extent_size, const blob *blobby, parsed_blob *pblobby) { + debug_assert(blobby->format == BLOB_FORMAT); pblobby->base = blobby; uint64 remainder = blobby->length; @@ -72,6 +89,7 @@ blob_length(slice sblobby) { const blob *blobby = slice_data(sblobby); debug_assert(sizeof(*blobby) <= slice_length(sblobby)); + debug_assert(blobby->format == BLOB_FORMAT); return blobby->length; } @@ -137,6 +155,12 @@ blob_page_iterator_init(cache *cc, || mode == BLOB_PAGE_ITERATOR_MODE_NO_PREFETCH || mode == BLOB_PAGE_ITERATOR_MODE_ALLOC); + const blob *blobby; + platform_status rc = blob_get_descriptor(sblobby, &blobby); + if (!SUCCESS(rc)) { + return rc; + } + iter->cc = cc; iter->mode = mode; iter->extent_size = cache_extent_size(cc); @@ -144,8 +168,7 @@ blob_page_iterator_init(cache *cc, iter->offset = offset; iter->page = NULL; - parse_blob( - iter->extent_size, iter->page_size, slice_data(sblobby), &iter->pblob); + parse_blob(iter->extent_size, iter->page_size, blobby, &iter->pblob); debug_assert(offset <= iter->pblob.base->length); @@ -264,6 +287,86 @@ blob_page_iterator_advance_page(blob_page_iterator *iter) blob_page_iterator_advance_bytes(iter, iter->fragment.length); } +platform_status +blob_validate(cache *cc, slice sblob) +{ + if (cc == NULL) { + return STATUS_BAD_PARAM; + } + + const blob *blobby; + platform_status rc = blob_get_descriptor(sblob, &blobby); + if (!SUCCESS(rc)) { + return rc; + } + checksum128 expected = blobby->checksum; + + XXH3_state_t *checksum_state = XXH3_createState(); + if (checksum_state == NULL) { + return STATUS_NO_MEMORY; + } + if (XXH3_128bits_reset_withSeed(checksum_state, BLOB_CHECKSUM_SEED) + != XXH_OK) + { + XXH3_freeState(checksum_state); + return STATUS_INVALID_STATE; + } + + blob_page_iterator iter; + /* + * Recovery can encounter a descriptor whose log page reached disk while + * one of the blob pages did not. Do not prefetch ahead of the readability + * check below: cache_get() is deliberately strict and a short read is a + * cache invariant violation, whereas an incomplete blob is ordinary crash + * truncation that validation must report to the log iterator. + */ + rc = blob_page_iterator_init( + cc, &iter, sblob, 0, BLOB_PAGE_ITERATOR_MODE_NO_PREFETCH); + if (!SUCCESS(rc)) { + XXH3_freeState(checksum_state); + return rc; + } + + while (!blob_page_iterator_at_end(&iter)) { + bool32 readable; + rc = cache_range_is_readable( + cc, iter.fragment.addr, iter.page_size, &readable); + if (!SUCCESS(rc)) { + goto out; + } + if (!readable) { + rc = STATUS_IO_ERROR; + goto out; + } + + uint64 offset; + slice data; + rc = blob_page_iterator_get_curr(&iter, &offset, &data); + if (!SUCCESS(rc)) { + goto out; + } + + if (XXH3_128bits_update( + checksum_state, slice_data(data), slice_length(data)) + != XXH_OK) + { + rc = STATUS_INVALID_STATE; + goto out; + } + blob_page_iterator_advance_page(&iter); + } + + checksum128 actual = XXH3_128bits_digest(checksum_state); + if (!platform_checksum_is_equal(actual, expected)) { + rc = STATUS_IO_ERROR; + } + +out: + blob_page_iterator_deinit(&iter); + XXH3_freeState(checksum_state); + return rc; +} + platform_status blob_materialize(cache *cc, slice sblobby, @@ -271,13 +374,17 @@ blob_materialize(cache *cc, uint64 end, writable_buffer *result) { - const blob *blobby = slice_data(sblobby); + const blob *blobby; + platform_status rc = blob_get_descriptor(sblobby, &blobby); + if (!SUCCESS(rc)) { + return rc; + } if (end < start || blobby->length < end) { return STATUS_BAD_PARAM; } - platform_status rc = writable_buffer_resize(result, end - start); + rc = writable_buffer_resize(result, end - start); if (!SUCCESS(rc)) { return rc; } @@ -310,26 +417,131 @@ blob_materialize(cache *cc, } platform_status -blob_sync(cache *cc, slice sblob) +blob_materialize_full(cache *cc, slice sblob, writable_buffer *result) { - blob_page_iterator itor; - platform_status rc = blob_page_iterator_init( - cc, &itor, sblob, 0, BLOB_PAGE_ITERATOR_MODE_NO_PREFETCH); + const blob *blobby; + platform_status rc = blob_get_descriptor(sblob, &blobby); if (!SUCCESS(rc)) { return rc; } - while (!blob_page_iterator_at_end(&itor)) { - uint64 offset; - slice result; - rc = blob_page_iterator_get_curr(&itor, &offset, &result); + return blob_materialize(cc, sblob, 0, blobby->length, result); +} + +/* + * Record one reference for the extent containing addr, unless something already + * has. See blob_recover_allocations(). + */ +static platform_status +blob_recover_extent(cache *cc, uint64 addr) +{ + allocator *al = cache_get_allocator(cc); + uint64 base = + allocator_config_extent_base_addr(allocator_get_config(al), addr); + + if (allocator_get_refcount(al, base) != AL_FREE) { + return STATUS_OK; + } + return allocator_recovery_record_reference(al, base, PAGE_TYPE_BLOB); +} + +platform_status +blob_recover_allocations(cache *cc, slice sblob) +{ + const blob *blobby; + platform_status rc = blob_get_descriptor(sblob, &blobby); + if (!SUCCESS(rc)) { + return rc; + } + + uint64 extent_size = cache_extent_size(cc); + uint64 page_size = cache_page_size(cc); + parsed_blob pblob; + + parse_blob(extent_size, page_size, blobby, &pblob); + + for (uint64 i = 0; i < pblob.num_extents; i++) { + rc = blob_recover_extent(cc, pblob.base->addrs[i]); + if (!SUCCESS(rc)) { + return rc; + } + } + /* + * The tail lives in up to three page-aligned fragments, which may sit in an + * extent this blob does not otherwise occupy -- and typically one it shares. + */ + for (uint64 i = 0; i < ARRAY_SIZE(pblob.leftovers); i++) { + if (pblob.leftovers[i].length == 0) { + break; + } + rc = blob_recover_extent(cc, pblob.leftovers[i].addr); + if (!SUCCESS(rc)) { + return rc; + } + } + return STATUS_OK; +} + +platform_status +blob_writeback(cache *cc, slice sblob, writeback_set *set) +{ + const blob *blobby; + platform_status rc = blob_get_descriptor(sblob, &blobby); + if (!SUCCESS(rc)) { + return rc; + } + + uint64 extent_size = cache_extent_size(cc); + uint64 page_size = cache_page_size(cc); + parsed_blob pblob; + + parse_blob(extent_size, page_size, blobby, &pblob); + + for (uint64 i = 0; i < pblob.num_extents; i++) { + if (set != NULL) { + rc = + writeback_set_add_extent(set, pblob.base->addrs[i], PAGE_TYPE_BLOB); + } else { + rc = cache_writeback_extent( + cc, pblob.base->addrs[i], PAGE_TYPE_BLOB, NULL); + } if (!SUCCESS(rc)) { + return rc; + } + } + + for (uint64 i = 0; i < ARRAY_SIZE(pblob.leftovers); i++) { + const parsed_blob_entry *tail = &pblob.leftovers[i]; + if (tail->length == 0) { break; } - cache_page_writeback(cc, itor.page, FALSE, PAGE_TYPE_BLOB); - blob_page_iterator_advance_page(&itor); + + uint64 byte_addr = tail->addr; + uint64 remaining = tail->length; + while (remaining > 0) { + uint64 page_offset = byte_addr % page_size; + uint64 page_addr = byte_addr - page_offset; + uint64 bytes_on_page = MIN(remaining, page_size - page_offset); + + page_handle *page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_BLOB); + if (page == NULL) { + return STATUS_IO_ERROR; + } + + if (set != NULL) { + rc = writeback_set_add_page(set, page, PAGE_TYPE_BLOB); + } else { + rc = cache_writeback_page(cc, page, PAGE_TYPE_BLOB, NULL); + } + cache_unget(cc, page); + if (!SUCCESS(rc)) { + return rc; + } + + byte_addr += bytes_on_page; + remaining -= bytes_on_page; + } } - blob_page_iterator_deinit(&itor); - return rc; + return STATUS_OK; } diff --git a/src/blob.h b/src/blob.h index 7c737a74..71b6804a 100644 --- a/src/blob.h +++ b/src/blob.h @@ -3,14 +3,31 @@ #pragma once +#include + +#include "platform_hash.h" #include "cache.h" #include "util.h" +#include "writeback_set.h" + +#define BLOB_FORMAT UINT16_C(1) +#define BLOB_CHECKSUM_SEED UINT64_C(0x424C4F424353554D) typedef struct ONDISK blob { - uint64 length; - uint64 addrs[]; + uint64 length; + checksum128 checksum; + uint16 format; + uint64 addrs[]; } blob; +_Static_assert(offsetof(blob, length) == 0, "blob length layout changed"); +_Static_assert(offsetof(blob, checksum) == 8, "blob checksum layout changed"); +_Static_assert(offsetof(blob, format) == 24, "blob format layout changed"); +_Static_assert(sizeof(((blob *)0)->format) == sizeof(uint16), + "blob format must be uint16"); +_Static_assert(offsetof(blob, addrs) == 26, "blob address layout changed"); +_Static_assert(sizeof(blob) == 26, "blob header layout changed"); + typedef struct parsed_blob_entry { uint64 addr; uint64 length; @@ -58,6 +75,16 @@ parse_blob(uint64 extent_size, uint64 blob_length(slice sblob); +/* + * Read and checksum all logical bytes referenced by sblob. Every backing page + * must be readable from the I/O address space; this is intended for recovery, + * after the blob's writeback has completed. Returns STATUS_INVALID_STATE for + * an invalid descriptor format and STATUS_IO_ERROR when a page is absent or the + * stored checksum does not match. + */ +platform_status +blob_validate(cache *cc, slice sblob); + platform_status blob_page_iterator_init(cache *cc, blob_page_iterator *iter, @@ -89,11 +116,36 @@ blob_materialize(cache *cc, uint64 end, writable_buffer *result); -static inline platform_status -blob_materialize_full(cache *cc, slice sblob, writable_buffer *result) -{ - return blob_materialize(cc, sblob, 0, blob_length(sblob), result); -} - platform_status -blob_sync(cache *cc, slice sblob); +blob_materialize_full(cache *cc, slice sblob, writable_buffer *result); + +/* + * Issue writeback of every page of the blob, recording each in `set` so the + * caller can later wait for them. Does not wait and does not make anything + * durable; `set` may be NULL to issue and forget. + * + * A caller that treats a blob as part of some larger durable unit must pass a + * set: the blob holds the record's value, so a unit declared durable without + * it would replay a record whose value never reached the device. + */ +platform_status +blob_writeback(cache *cc, slice sblob, writeback_set *set); + +/* + * Record the allocator references this blob's storage holds, for a crash- + * recovery rebuild. Call between allocator_recovery_begin() and + * allocator_recovery_finish(). + * + * Reads nothing. A blob carries the addresses of its own storage inline (see + * struct blob), so the extents can be named without touching a page -- which is + * what makes this usable during a rebuild, when reading a page whose extent is + * not yet marked allocated is exactly what is forbidden. + * + * Records at most one reference per extent, skipping any that already has one. + * That is the correct count and not merely deduplication: an extent gets a + * single reference when its mini allocator hands it out, and blobs share + * extents -- one holds the tails of many -- so counting per blob would leave + * extents referenced several times over and never freed. + */ +platform_status +blob_recover_allocations(cache *cc, slice sblob); diff --git a/src/blob_build.c b/src/blob_build.c index c97aff82..ea453b31 100644 --- a/src/blob_build.c +++ b/src/blob_build.c @@ -4,6 +4,13 @@ #include "blob_build.h" #include "poison.h" +static checksum128 +checksum_blob_data(slice data) +{ + const void *bytes = slice_length(data) == 0 ? "" : slice_data(data); + return platform_checksum128(bytes, slice_length(data), BLOB_CHECKSUM_SEED); +} + static platform_status allocate_leftover_entries(const blob_build_config *cfg, cache *cc, @@ -120,8 +127,10 @@ build_blob_table(const blob_build_config *cfg, return rc; } - blob *blobby = writable_buffer_data(result); - blobby->length = data_len; + blob *blobby = writable_buffer_data(result); + blobby->length = data_len; + blobby->checksum = (checksum128){0}; + blobby->format = BLOB_FORMAT; for (uint64 i = 0; i < num_extents; i++) { uint64 alloced_page = mini_alloc_extent(mini, cfg->extent_batch, NULL); @@ -175,6 +184,10 @@ blob_build(const blob_build_config *cfg, out: blob_page_iterator_deinit(&iter); + if (SUCCESS(rc)) { + blob *blobby = writable_buffer_data(result); + blobby->checksum = checksum_blob_data(data); + } return rc; } @@ -191,8 +204,8 @@ clone_blob_table(const blob_build_config *cfg, return rc; } - blob *blobby = writable_buffer_data(result); - blobby->length = pblobby->base->length; + blob *blobby = writable_buffer_data(result); + memcpy(blobby, pblobby->base, sizeof(*blobby)); for (uint64 i = 0; i < pblobby->num_extents; i++) { blobby->addrs[i] = pblobby->base->addrs[i]; @@ -220,9 +233,16 @@ blob_clone(const blob_build_config *cfg, slice sblob, writable_buffer *result) { + if (slice_length(sblob) < sizeof(blob)) { + return STATUS_INVALID_STATE; + } + uint64 extent_size = cache_extent_size(cc); uint64 page_size = cache_page_size(cc); const blob *blobby = slice_data(sblob); + if (blobby->format != BLOB_FORMAT) { + return STATUS_INVALID_STATE; + } parsed_blob pblobby; parse_blob(extent_size, page_size, blobby, &pblobby); diff --git a/src/btree.c b/src/btree.c index 9504aba3..a3a1849c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -645,6 +645,14 @@ btree_record_insert_msg_blob(btree_insert_results *results, return success ? STATUS_OK : STATUS_NO_MEMORY; } +static inline void +btree_insert_invoke_callback(btree_insert_results *results) +{ + if (results->callback != NULL) { + results->callback(results->callback_arg); + } +} + platform_status btree_create_leaf_incorporate_spec(const btree_config *cfg, cache *cc, @@ -972,7 +980,7 @@ btree_split_leaf_build_right_node(const btree_config *cfg, // IN leaf_incorporate_spec *spec, // IN leaf_splitting_plan plan, // IN btree_hdr *right_hdr, - uint64 *generation) // IN/OUT + btree_insert_results *results) // IN/OUT { /* Build the right node. */ memmove(right_hdr, left_hdr, sizeof(*right_hdr)); @@ -993,8 +1001,9 @@ btree_split_leaf_build_right_node(const btree_config *cfg, // IN if (!plan.insertion_goes_left) { spec->idx -= plan.split_idx; + btree_insert_invoke_callback(results); bool32 incorporated = btree_try_perform_leaf_incorporate_spec( - cfg, right_hdr, spec, generation); + cfg, right_hdr, spec, &results->leaf_generation); platform_assert(incorporated); } } @@ -1323,6 +1332,22 @@ btree_inc_ref(cache *cc, const btree_config *cfg, uint64 root_addr) mini_inc_ref(cc, meta_page_addr); } +/* + * Record the reference a holder has on this branch during a crash-recovery + * rebuild, enumerating the branch's extents if this is the first one to reach + * it. The recovery counterpart of btree_inc_ref(); see + * mini_recover_references(). + */ +platform_status +btree_recover_allocations(cache *cc, + const btree_config *cfg, + uint64 root_addr, + page_type type) +{ + return mini_recover_references( + cc, btree_root_to_meta_addr(cfg, root_addr, 0), type); +} + bool32 btree_dec_ref(cache *cc, const btree_config *cfg, @@ -1510,19 +1535,15 @@ btree_split_child_leaf(cache *cc, } /* p: unlocked, c: write, rc: write, cn: unlocked */ - btree_split_leaf_build_right_node(cfg, - child->hdr, - child->addr, - spec, - plan, - right_child.hdr, - &results->leaf_generation); + btree_split_leaf_build_right_node( + cfg, child->hdr, child->addr, spec, plan, right_child.hdr, results); btree_node_full_unlock(cc, cfg, &right_child); /* p: unlocked, c: write, rc: unlocked, cn: unlocked */ btree_split_leaf_cleanup_left_node( cfg, scratch, child->hdr, spec, plan, right_child.addr); if (plan.insertion_goes_left) { + btree_insert_invoke_callback(results); bool32 incorporated = btree_try_perform_leaf_incorporate_spec( cfg, child->hdr, spec, &results->leaf_generation); platform_assert(incorporated); @@ -1587,6 +1608,7 @@ btree_defragment_or_split_child_leaf(cache *cc, return rc; } btree_defragment_leaf(cfg, scratch, child->hdr, spec); + btree_insert_invoke_callback(results); bool32 incorporated = btree_try_perform_leaf_incorporate_spec( cfg, child->hdr, spec, &results->leaf_generation); platform_assert(incorporated); @@ -1907,6 +1929,7 @@ btree_insert(cache *cc, // IN destroy_leaf_incorporate_spec(&spec); return rc; } + btree_insert_invoke_callback(results); bool32 incorporated = btree_try_perform_leaf_incorporate_spec( cfg, root_node.hdr, &spec, &results->leaf_generation); platform_assert(incorporated); @@ -2127,6 +2150,7 @@ btree_insert(cache *cc, // IN destroy_leaf_incorporate_spec(&spec); return rc; } + btree_insert_invoke_callback(results); bool32 incorporated = btree_try_perform_leaf_incorporate_spec( cfg, child_node.hdr, &spec, &results->leaf_generation); platform_assert(incorporated); diff --git a/src/btree.h b/src/btree.h index 48795fb0..96f1fe0e 100644 --- a/src/btree.h +++ b/src/btree.h @@ -223,10 +223,19 @@ typedef struct btree_pack_req { uint64 message_bytes; // total size of msgs in tuples of the output tree } btree_pack_req; +typedef void (*btree_insert_callback_fn)(void *arg); + typedef struct btree_insert_results { lookup_result *old_result_buffer; // optional, not owned uint64 leaf_generation; merge_accumulator msg_blob; + /* + * Optional, infallible callback invoked exactly once on a successful insert + * while the final target leaf is write-locked, immediately before the + * guaranteed logical incorporation. + */ + btree_insert_callback_fn callback; + void *callback_arg; } btree_insert_results; static inline void @@ -236,6 +245,17 @@ btree_insert_results_init(btree_insert_results *results, results->old_result_buffer = old_result_buffer; results->leaf_generation = 0; merge_accumulator_init(&results->msg_blob, PROCESS_PRIVATE_HEAP_ID); + results->callback = NULL; + results->callback_arg = NULL; +} + +static inline void +btree_insert_results_set_callback(btree_insert_results *results, + btree_insert_callback_fn callback, + void *callback_arg) +{ + results->callback = callback; + results->callback_arg = callback_arg; } static inline void @@ -244,6 +264,8 @@ btree_insert_results_deinit(btree_insert_results *results) merge_accumulator_deinit(&results->msg_blob); results->old_result_buffer = NULL; results->leaf_generation = 0; + results->callback = NULL; + results->callback_arg = NULL; } platform_status @@ -272,6 +294,16 @@ btree_dec_ref(cache *cc, uint64 root_addr, page_type type); +/* + * Rebuild the allocator references this branch holds after a crash. The + * recovery counterpart of btree_inc_ref(); see mini_recover_references(). + */ +platform_status +btree_recover_allocations(cache *cc, + const btree_config *cfg, + uint64 root_addr, + page_type type); + void btree_node_unget(cache *cc, const btree_config *cfg, btree_node *node); diff --git a/src/cache.h b/src/cache.h index 598df0f5..d10f1833 100644 --- a/src/cache.h +++ b/src/cache.h @@ -128,15 +128,70 @@ typedef page_handle *(*page_get_fn)(cache *cc, bool32 blocking, page_type type); typedef bool32 (*page_try_claim_fn)(cache *cc, page_handle *page); -typedef void (*page_writeback_fn)(cache *cc, - page_handle *page, - bool32 is_blocking, - page_type type); -typedef void (*extent_writeback_fn)(cache *cc, - uint64 addr, - uint64 *pages_outstanding); + +/* + * ---- Writeback requests ---- + * + * A writeback request is the receipt for one issued writeback: it names what + * was written and identifies the page's dirty interval at the moment the write + * was handed to the I/O layer. Present it to cache_writeback_get_status() to + * learn whether that write has completed. + * + * The interval identity is what makes a request meaningful, so it is produced + * by the call that issues the write -- where it can be read atomically with the + * CC_WRITEBACK transition -- rather than by a free-standing query. + * + * A request holds only the address, so it survives eviction of the page it + * names and the caller need not hold a reference: a page that is no longer + * resident was necessarily written back before it was evicted. + * + * gen == 0 means "nothing to wait for": the page was already clean when the + * writeback was requested. + * + * A caller that only wants the write issued, and will never ask whether it + * completed, may pass NULL instead of a request. + */ +typedef struct cache_writeback_request { + uint64 addr; // page addr, or the base addr of an extent + uint64 gen; // dirty interval the write covers; 0 == nothing issued + bool32 is_extent; // whether addr names an extent rather than one page +} cache_writeback_request; + +typedef enum cache_writeback_status { + /* The write is still outstanding. Poll cache_cleanup() and re-check. */ + CACHE_WRITEBACK_PENDING, + /* The write completed: the contents as of the request reached the device. + * Note this is completion, NOT durability -- see cache_durable_barrier(). */ + CACHE_WRITEBACK_COMPLETE, + /* The write completed, but the page has since been dirtied again. The + * request is satisfied; callers that do not expect concurrent writers to + * their pages should treat this as a bug in their own locking. */ + CACHE_WRITEBACK_REDIRTIED, + /* + * The write FAILED: these contents did not reach the device, and the request + * will never be satisfied without a successful retry. Polling again does + * not help -- the cache retries failed writes on its own schedule, so a + * caller that wants to keep waiting must be prepared to wait indefinitely; + * one that needs durability now must propagate the failure. + * + * For an extent request this outranks REDIRTIED but not PENDING, so by the + * time it is reported no write on the extent is still in flight. + */ + CACHE_WRITEBACK_FAILED, +} cache_writeback_status; + +typedef platform_status (*page_writeback_fn)(cache *cc, + page_handle *page, + page_type type, + cache_writeback_request *req); +typedef platform_status (*extent_writeback_fn)(cache *cc, + uint64 addr, + page_type type, + cache_writeback_request *req); +typedef cache_writeback_status ( + *writeback_get_status_fn)(cache *cc, const cache_writeback_request *req); typedef void (*page_prefetch_fn)(cache *cc, uint64 addr, page_type type); -typedef int (*evict_fn)(cache *cc, bool32 ignore_pinned); +typedef platform_status (*evict_fn)(cache *cc, bool32 ignore_pinned); typedef bool32 (*page_addr_pred_fn)(cache *cc, uint64 addr); typedef void (*page_addr_fn)(cache *cc, uint64 addr); typedef void (*validate_page_fn)(cache *cc, page_handle *page, uint64 addr); @@ -145,6 +200,10 @@ typedef uint32 (*count_dirty_fn)(cache *cc); typedef uint16 (*page_get_read_ref_fn)(cache *cc, page_handle *page); typedef bool32 (*cache_present_fn)(cache *cc, page_handle *page); typedef void (*enable_sync_get_fn)(cache *cc, bool32 enabled); +typedef platform_status (*cache_range_is_readable_fn)(cache *cc, + uint64 addr, + uint64 bytes, + bool32 *readable); typedef allocator *(*get_allocator_fn)(const cache *cc); typedef cache_config *(*cache_config_fn)(const cache *cc); typedef void (*cache_print_fn)(platform_log_handle *log_handle, cache *cc); @@ -163,36 +222,38 @@ typedef struct cache_ops { page_get_async_fn page_get_async; page_get_async_state_result_fn page_get_async_result; - page_generic_fn page_unget; - page_try_claim_fn page_try_claim; - page_generic_fn page_unclaim; - page_generic_fn page_lock; - page_generic_fn page_unlock; - page_prefetch_fn page_prefetch; - page_prefetch_fn page_prefetch_page; - page_generic_fn page_pin; - page_generic_fn page_unpin; - page_writeback_fn page_writeback; - extent_writeback_fn extent_writeback; - cache_generic_void_fn flush; - cache_generic_status_fn writeback_dirty; - cache_generic_status_fn durable_barrier; - evict_fn evict; - cache_generic_void_fn cleanup; - page_addr_pred_fn in_use; - page_addr_fn assert_ungot; - cache_generic_void_fn assert_free; - validate_page_fn validate_page; - cache_present_fn cache_present; - cache_print_fn print; - cache_print_fn print_stats; - io_stats_fn io_stats; - cache_generic_void_fn reset_stats; - count_dirty_fn count_dirty; - page_get_read_ref_fn page_get_read_ref; - enable_sync_get_fn enable_sync_get; - get_allocator_fn get_allocator; - cache_config_fn get_config; + page_generic_fn page_unget; + page_try_claim_fn page_try_claim; + page_generic_fn page_unclaim; + page_generic_fn page_lock; + page_generic_fn page_unlock; + page_prefetch_fn page_prefetch; + page_prefetch_fn page_prefetch_page; + page_generic_fn page_pin; + page_generic_fn page_unpin; + page_writeback_fn page_writeback; + extent_writeback_fn extent_writeback; + writeback_get_status_fn writeback_get_status; + cache_generic_void_fn flush; + cache_generic_status_fn writeback_dirty; + cache_generic_status_fn durable_barrier; + evict_fn evict; + cache_generic_void_fn cleanup; + page_addr_pred_fn in_use; + page_addr_fn assert_ungot; + cache_generic_void_fn assert_free; + validate_page_fn validate_page; + cache_present_fn cache_present; + cache_print_fn print; + cache_print_fn print_stats; + io_stats_fn io_stats; + cache_generic_void_fn reset_stats; + count_dirty_fn count_dirty; + page_get_read_ref_fn page_get_read_ref; + enable_sync_get_fn enable_sync_get; + cache_range_is_readable_fn range_is_readable; + get_allocator_fn get_allocator; + cache_config_fn get_config; } cache_ops; // To sub-class cache, make a cache your first field; @@ -463,54 +524,89 @@ cache_unpin(cache *cc, page_handle *page) /* *----------------------------------------------------------------------------- - * cache_page_writeback + * cache_writeback_page + * + * Asynchronously issues writeback of the page and, if req is non-NULL, fills + * in *req, the receipt for that write. This does NOT make the page durable; it + * only hands the write to the I/O layer (no device-cache flush). Use + * cache_writeback_get_status() to learn when it completes and + * cache_durable_barrier() to make it durable. * - * Issues writeback of the page to disk. This does NOT make the page durable; - * it only hands the write to the I/O layer (no device-cache flush). + * req may be NULL if the caller will never ask whether the write completed. + * The returned status is still worth checking even then; see below. * - * With is_blocking == FALSE the writeback is issued asynchronously and this - * returns without waiting for completion; there is no per-call way to observe - * when it finishes. With is_blocking == TRUE the page is written synchronously - * and the write has completed by the time this returns. + * Does not block. If a writeback of this page is already in flight -- issued by + * the pressure cleaner, say -- nothing further is issued and *req names that + * in-flight interval, so the caller waits on it exactly as it would on its own + * write. If the page is already clean, req->gen is 0: nothing to wait for. + * + * Returns STATUS_BUSY if the page is dirty but not writeback-able (locked or + * claimed). Callers must treat that as a failure to make the page durable: no + * write was issued and *req cannot report one. + * + * If an earlier write of this page failed, this claims the retry of it, so the + * returned request covers a fresh attempt rather than reporting the old + * failure. *----------------------------------------------------------------------------- */ -static inline void -cache_page_writeback(cache *cc, - page_handle *page, - bool32 is_blocking, - page_type type) +static inline platform_status +cache_writeback_page(cache *cc, + page_handle *page, + page_type type, + cache_writeback_request *req) { - return cc->ops->page_writeback(cc, page, is_blocking, type); + // Absorb the optional request here so that every cache implementation may + // assume it was given somewhere to write the receipt. + cache_writeback_request scratch; + return cc->ops->page_writeback(cc, page, type, req ? req : &scratch); } /* *----------------------------------------------------------------------------- - * cache_extent_writeback + * cache_writeback_extent * - * Asynchronously issues writeback of the extent beginning at addr. This does - * NOT make the extent durable; it only hands the writes to the I/O layer (no - * device-cache flush) and returns without waiting for completion. + * As cache_writeback_page(), but for every page of the extent beginning at + * addr, coalesced into as few larger I/Os as the extent's residency allows. + * One request covers the whole extent: req->gen is the newest dirty interval + * among its pages, which is safe to compare every page against. As above, req + * may be NULL. * - * *pages_outstanding is immediately incremented by the number of pages - * issued for writeback (the non-clean pages of the extent); as writebacks - * complete, *pages_outstanding is decremented atomically. This counter is the - * only way to observe when the issued writebacks finish. + * Pages of the extent that are clean or not resident need no write and are + * skipped. Returns STATUS_BUSY if any page is dirty but not writeback-able; as + * with cache_writeback_page(), the caller must treat that as a failure, since + * that page's contents will not reach the device. * - * Assumes pages_outstanding is an aligned uint64, so (on x86) the caller - * can access it and observe its value atomically. + * Concurrent callers on the same extent are safe: only one can win a page's + * writeback, and the other's request names that same in-flight interval, so + * both observe its completion. + *----------------------------------------------------------------------------- + */ +static inline platform_status +cache_writeback_extent(cache *cc, + uint64 addr, + page_type type, + cache_writeback_request *req) +{ + cache_writeback_request scratch; + return cc->ops->extent_writeback(cc, addr, type, req ? req : &scratch); +} + +/* + *----------------------------------------------------------------------------- + * cache_writeback_get_status * - * TODO: What happens if two callers call cache_extent_writeback on the same - * extent? + * Whether the write named by *req has completed. Never blocks; a caller + * waiting for completion loops on this and cache_cleanup(), which reaps I/O + * completions on the calling thread and is therefore what makes the loop + * progress rather than merely spin. * - * All pages in the extent must be clean or cleanable. - * The page may not be in writeback, loading, or locked, or claimed, otherwise - * undefined behavior will occur. + * Completion is not durability: follow with cache_durable_barrier(). *----------------------------------------------------------------------------- */ -static inline void -cache_extent_writeback(cache *cc, uint64 addr, uint64 *pages_outstanding) +static inline cache_writeback_status +cache_writeback_get_status(cache *cc, const cache_writeback_request *req) { - cc->ops->extent_writeback(cc, addr, pages_outstanding); + return cc->ops->writeback_get_status(cc, req); } /* @@ -561,21 +657,21 @@ cache_durable_barrier(cache *cc) *----------------------------------------------------------------------------- * cache_evict * - * Evicts all the pages. - * Asserts that there are no pins (if ignore_pinned_pages is false), read - * locks, claims or write locks. - * Always returns 0. + * Waits for outstanding cache I/O and evicts every resident page. Dirty pages + * are not written: callers that need their contents must write them back first. + * The cache must otherwise be quiescent -- no read locks, claims, write locks, + * or concurrent cache users. * - * TODO: Does ignore_pinned_pages ignore the pages or the pinnedness of the - *pages? + * If ignore_pinned_pages is false, pinned pages make the call fail with + * STATUS_BUSY. If true, pinned pages themselves remain resident; this mode is + * useful to tests but is not a complete cache invalidation. * - * Test facility. - * This method is only used for testing, specifically in cache_test. - * TODO Should be deleted and replaced with destructing and constructing - * a fresh cache. + * A failure may follow partial progress: pages successfully evicted before the + * conflicting entry stay evicted. A quiescent caller may fix the conflict and + * call again. *----------------------------------------------------------------------------- */ -static inline int +static inline platform_status cache_evict(cache *cc, bool32 ignore_pinned_pages) { return cc->ops->evict(cc, ignore_pinned_pages); @@ -697,6 +793,17 @@ cache_io_stats(cache *cc, uint64 *read_bytes, uint64 *write_bytes) return cc->ops->io_stats(cc, read_bytes, write_bytes); } +/* + * Does the backing store contain the whole byte range [addr, addr + bytes)? + * This says only whether a strict read may be issued; callers must still + * validate the page format and contents. + */ +static inline platform_status +cache_range_is_readable(cache *cc, uint64 addr, uint64 bytes, bool32 *readable) +{ + return cc->ops->range_is_readable(cc, addr, bytes, readable); +} + /* *----------------------------------------------------------------------------- * cache_validate_page diff --git a/src/clockcache.c b/src/clockcache.c index bd59647b..a966acbc 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -37,6 +37,14 @@ // Number of batches that the cleaner hand is ahead of the evictor hand #define CC_CLEANER_GAP 512 +/* + * How many times a thread blocked in clockcache_get_write() will re-issue a + * failed writeback itself before falling back to waiting for the cleaner or a + * checkpoint pass. Bounds the work -- and the I/O -- done inside a lock + * acquisition when a device is failing persistently. + */ +#define CC_MAX_WRITEBACK_SYNC_RETRIES 8 + /* number of events to poll for during clockcache_wait */ #define CC_DEFAULT_MAX_IO_EVENTS 1 @@ -141,6 +149,16 @@ clockcache_print(platform_log_handle *log_handle, clockcache *cc); #define CC_LOADING (1u << 4) // page is actively being read from disk #define CC_WRITELOCKED (1u << 5) // write lock is held #define CC_CLAIMED (1u << 6) // claim is held +/* + * A writeback of this page failed. The page stays dirty and stays in + * CC_WRITEBACK, which leaves it un-claimable (try_set_writeback needs a + * cleanable status), un-evictable (try_evict needs CC_CLEAN) and un-writable + * (get_write excludes pages in writeback) -- exactly the exclusions a retry + * needs. It also pins dirty_generation, which is what keeps an outstanding + * cache_writeback_request naming this page valid. Cleared only by a thread that + * claims the retry. + */ +#define CC_WRITEBACK_ERROR (1u << 7) /* Common status flag combinations */ // free entry @@ -254,6 +272,35 @@ clockcache_dirty_begin(clockcache *cc, uint32 entry_number) * before CC_WRITEBACK is cleared: the intermediate CC_CLEAN|CC_WRITEBACK state * is not cleanable, so no thread can start a duplicate writeback in the gap. */ +/* + * The dirty interval the entry is currently in, or 0 if it is clean. + * + * Meaningful as a writeback receipt only when read while CC_WRITEBACK is set, + * which excludes the writer and so pins the interval; see + * clockcache_writeback_page(). + */ +static inline uint64 +clockcache_writeback_get_generation(clockcache *cc, uint32 entry_number) +{ + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + return __atomic_load_n(&entry->dirty_generation, __ATOMIC_RELAXED); +} + +/* + * Record a failed writeback. Deliberately leaves both dirty_generation and + * CC_WRITEBACK alone: the page must stay excluded from claiming, eviction and + * writing until a retry succeeds, and its generation must stay pinned so that + * an outstanding writeback receipt naming it stays valid. See + * CC_WRITEBACK_ERROR. + */ +static void +clockcache_dirty_fail_writeback(clockcache *cc, uint32 entry_number) +{ + debug_only uint32 was_error = + clockcache_set_flag(cc, entry_number, CC_WRITEBACK_ERROR); + debug_assert(!was_error, "writeback failed twice with no retry in between"); +} + static void clockcache_dirty_complete_writeback(clockcache *cc, uint32 entry_number) { @@ -681,6 +728,75 @@ clockcache_try_get_claim(clockcache *cc, uint32 entry_number) return GET_RC_SUCCESS; } +/* + *---------------------------------------------------------------------- + * clockcache_try_retry_writeback + * + * Claims the retry of a failed writeback, returning TRUE if we took it. + * Clearing CC_WRITEBACK_ERROR *is* the claim: only one thread can observe + * the bit set, so no exact-match compare-and-swap is needed and the other + * status bits are irrelevant. + * + * CC_WRITEBACK stays set throughout, so the page remains excluded from + * claiming, eviction and writing across the whole failure-and-retry + * window, and its dirty_generation stays pinned -- which is what keeps an + * outstanding cache_writeback_request naming it valid. The winner must + * issue the write, just as if it had won clockcache_try_set_writeback(). + *---------------------------------------------------------------------- + */ +static inline bool32 +clockcache_try_retry_writeback(clockcache *cc, uint32 entry_number) +{ + return clockcache_clear_flag(cc, entry_number, CC_WRITEBACK_ERROR) != 0; +} + +/* + *---------------------------------------------------------------------- + * clockcache_writeback_page_sync + * + * Writes a page back synchronously and does the dirty->clean bookkeeping + * inline that clockcache_write_callback() would otherwise do. + * + * Synchronous because the caller needs the write to have *landed* before + * it can proceed -- see clockcache_get_write(), which cannot take the + * write lock until the page leaves CC_WRITEBACK. Issuing asynchronously + * would just send it back to waiting. io_write() is a bare pwrite() and + * never re-enters the cache, so this is safe to call under the page lock. + * + * The caller must own the writeback claim -- CC_WRITEBACK set with the + * error bit clear -- having won clockcache_try_set_writeback() or + * clockcache_try_retry_writeback(). On failure the page goes back into the + * error state for someone else to retry. + *---------------------------------------------------------------------- + */ +static void +clockcache_writeback_page_sync(clockcache *cc, uint32 entry_number) +{ + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + uint64 addr = entry->page.disk_addr; + const threadid tid = platform_get_tid(); + + debug_assert(clockcache_test_flag(cc, entry_number, CC_WRITEBACK)); + debug_assert(!clockcache_test_flag(cc, entry_number, CC_WRITEBACK_ERROR)); + + if (cc->cfg->use_stats) { + cc->stats[tid].page_writes[entry->type]++; + cc->stats[tid].syncs_issued++; + } + + platform_status rc = + io_write(cc->io, entry->page.data, clockcache_page_size(cc), addr); + if (SUCCESS(rc)) { + clockcache_dirty_complete_writeback(cc, entry_number); + } else { + platform_error_log("clockcache_writeback_page_sync: io_write failed for " + "addr %lu: %s\n", + addr, + platform_status_to_string(rc)); + clockcache_dirty_fail_writeback(cc, entry_number); + } +} + /* *---------------------------------------------------------------------- * clockcache_get_write @@ -721,8 +837,35 @@ clockcache_get_write(clockcache *cc, uint32 entry_number) * background threads. */ debug_assert(clockcache_get_ref(cc, entry_number, tid) >= 1); - // Wait for flushing to finish + /* + * Wait for flushing to finish. + * + * A page whose writeback failed stays in CC_WRITEBACK (see + * CC_WRITEBACK_ERROR), so we cannot simply proceed: granting the write lock + * would let dirty_generation advance, and an outstanding + * cache_writeback_request naming this page would then read as REDIRTIED -- + * i.e. satisfied -- for contents that never reached the device. + * + * We can, however, re-issue the write ourselves instead of waiting for a + * cleaner or checkpoint pass to happen along. We are already blocked on + * this exact page, so a synchronous write costs nothing we were not already + * paying, and it removes the dependency on another actor showing up. Doing + * so does not grant the lock early, so the receipt stays honest either way. + */ + uint64 sync_retries = 0; while (clockcache_test_flag(cc, entry_number, CC_WRITEBACK)) { + if (sync_retries < CC_MAX_WRITEBACK_SYNC_RETRIES + && clockcache_try_retry_writeback(cc, entry_number)) + { + sync_retries++; + platform_error_log( + "clockcache_get_write: re-issuing failed writeback of addr %lu " + "(attempt %lu)\n", + clockcache_get_entry(cc, entry_number)->page.disk_addr, + sync_retries); + clockcache_writeback_page_sync(cc, entry_number); + continue; + } clockcache_wait(cc); } @@ -773,7 +916,12 @@ clockcache_try_get_write(clockcache *cc, uint32 entry_number) debug_assert(!was_writing); debug_assert(!clockcache_test_flag(cc, entry_number, CC_LOADING)); - // if flushing, then bail + /* + * If flushing, then bail. This also covers a page whose writeback failed and + * is awaiting a retry, since such a page stays in CC_WRITEBACK; the caller + * will keep retrying until a writeback retry succeeds. See + * clockcache_get_write() for why we must not grant the lock instead. + */ if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK)) { rc = GET_RC_FLUSHING; goto failed; @@ -848,6 +996,21 @@ clockcache_ok_to_writeback(clockcache *cc, * compare-and- swaps, so we retry as long as the status remains one of the * cleanable states rather than spuriously failing on a page that stayed * cleanable. + * + * INVARIANT: a claim cannot be retracted. Once CC_WRITEBACK is set, a + * write must be issued and must run to completion, because another thread + * may already hold a CC_WRITEBACK_INFLIGHT receipt naming this page's + * dirty interval (see cache_writeback_page()). Clearing CC_WRITEBACK + * without writing would make that thread's next status poll read the page + * as REDIRTIED, which is indistinguishable from a completed write followed + * by a legitimate re-dirty -- a silent durability violation. So every + * failure between here and io_async_run() is fatal. + * + * Handling such a failure without crashing needs a state a waiter can + * observe and react to -- an error flag that some thread later claims a + * retry on, plus a status the waiter can fail on rather than hang. That is + * a bigger change, and it belongs with handling write-completion errors + * (clockcache_write_callback() asserts on those today), not ahead of it. *---------------------------------------------------------------------- */ static inline bool32 @@ -885,11 +1048,112 @@ clockcache_try_set_writeback(clockcache *cc, } } -typedef struct async_io_state { +/* + *---------------------------------------------------------------------- + * clockcache_try_claim_writeback + * + * Takes ownership of writing a page back, either freshly (it is dirty and + * cleanable) or as a retry (an earlier write of it failed). Both outcomes + * leave the page in the same state -- CC_WRITEBACK set and owned by us -- + * so a caller that goes on to issue the I/O need not tell them apart. + * That is what lets clockcache_batch_start_writeback()'s extent coalescing + * treat a retried page and a freshly dirtied one as interchangeable, and + * keeps its backward/forward walk unchanged. + * + * A retry is deliberately not gated on with_access: that bit is a hint + * about eviction cost, whereas a failed write must be re-issued regardless + * in order to release whatever is blocked behind it. + *---------------------------------------------------------------------- + */ +static inline bool32 +clockcache_try_claim_writeback(clockcache *cc, + uint32 entry_number, + bool32 with_access) +{ + return (clockcache_ok_to_writeback(cc, entry_number, with_access) + && clockcache_try_set_writeback(cc, entry_number, with_access)) + || clockcache_try_retry_writeback(cc, entry_number); +} + +struct async_io_state { clockcache *cc; - uint64 *outstanding_pages; io_async_state_buffer iostate; -} async_io_state; +}; + +/* Bounded by the width of the bitvector that tracks the reserve. */ +#define CC_IO_STATE_POOL_SIZE (64) +_Static_assert(CC_IO_STATE_POOL_SIZE <= 64, + "clockcache::io_state_pool_free is a uint64"); + +/* + *---------------------------------------------------------------------- + * clockcache_io_state_acquire -- + * + * Obtains an async_io_state with which to issue a writeback. Never fails. + * + * Tries the heap first, so the reserve stays untouched in the common case + * and its fixed size never caps writeback concurrency. Falls back to the + * reserve only when the heap is exhausted, which is exactly when a + * writeback most needs to proceed. + * + * An empty reserve means all CC_IO_STATE_POOL_SIZE states are checked out, + * i.e. that many writebacks are in flight, so waiting for one to land is + * guaranteed to make progress. clockcache_wait() is what makes the wait + * terminate: it reaps completions on this thread rather than relying on + * some other thread happening to do it. + *---------------------------------------------------------------------- + */ +static async_io_state * +clockcache_io_state_acquire(clockcache *cc) +{ + while (TRUE) { + async_io_state *state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); + if (state != NULL) { + return state; + } + + uint64 free_mask = + __atomic_load_n(&cc->io_state_pool_free, __ATOMIC_RELAXED); + while (free_mask != 0) { + uint64 slot = __builtin_ctzl(free_mask); + uint64 bit = 1ULL << slot; + // Clearing the bit is the claim: whoever observes it set was the one + // who cleared it. + uint64 prev = __sync_fetch_and_and(&cc->io_state_pool_free, ~bit); + if (prev & bit) { + return &cc->io_state_pool[slot]; + } + free_mask = prev & ~bit; + } + + clockcache_wait(cc); + } +} + +/* + *---------------------------------------------------------------------- + * clockcache_io_state_release -- + * + * Returns a state obtained from clockcache_io_state_acquire(). Whether it + * came from the reserve is decided by its address, so callers need not + * remember where it came from. + *---------------------------------------------------------------------- + */ +static void +clockcache_io_state_release(clockcache *cc, async_io_state *state) +{ + if (cc->io_state_pool <= state + && state < cc->io_state_pool + CC_IO_STATE_POOL_SIZE) + { + uint64 slot = (uint64)(state - cc->io_state_pool); + uint64 bit = 1ULL << slot; + debug_only uint64 prev = + __sync_fetch_and_or(&cc->io_state_pool_free, bit); + debug_assert(!(prev & bit), "double release of io state slot %lu", slot); + } else { + platform_free(PROCESS_PRIVATE_HEAP_ID, state); + } +} static void clockcache_write_callback(void *wbs) @@ -906,7 +1170,6 @@ clockcache_write_callback(void *wbs) platform_error_log("clockcache_write_callback: async write failed: %s\n", platform_status_to_string(rc)); } - platform_assert_status_ok(rc); const struct iovec *iovec; uint64 count; @@ -933,35 +1196,21 @@ clockcache_write_callback(void *wbs) entry_number, addr); - clockcache_dirty_complete_writeback(cc, entry_number); - } - - if (state->outstanding_pages) { - __sync_fetch_and_sub(state->outstanding_pages, count); + if (SUCCESS(rc)) { + clockcache_dirty_complete_writeback(cc, entry_number); + } else { + /* + * One status covers the whole request, so a failure fails every + * page in it. Some may in fact have landed; retrying is harmless. + */ + clockcache_dirty_fail_writeback(cc, entry_number); + } } io_async_state_deinit(state->iostate); - platform_free(PROCESS_PRIVATE_HEAP_ID, state); -} - -static void -clockcache_abort_writeback_range(clockcache *cc, - uint64 first_addr, - uint64 end_addr) -{ - uint64 page_size = clockcache_page_size(cc); - - for (uint64 addr = first_addr; addr < end_addr; addr += page_size) { - uint32 entry_number = clockcache_lookup(cc, addr); - platform_assert(entry_number != CC_UNMAPPED_ENTRY); - - debug_only uint32 was_writeback = - clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); - debug_assert(was_writeback); - } + clockcache_io_state_release(cc, state); } - /* *---------------------------------------------------------------------- * clockcache_batch_start_writeback -- @@ -986,7 +1235,6 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) uint64 end_entry_no = start_entry_no + CC_ENTRIES_PER_BATCH; clockcache_entry *entry, *next_entry; - platform_status result = STATUS_OK; debug_assert((tid < MAX_THREADS), "Invalid tid=%lu\n", tid); debug_assert(cc != NULL); @@ -1008,9 +1256,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) entry = &cc->entry[entry_no]; addr = entry->page.disk_addr; // test and test and set in the if condition - if (clockcache_ok_to_writeback(cc, entry_no, is_urgent) - && clockcache_try_set_writeback(cc, entry_no, is_urgent)) - { + if (clockcache_try_claim_writeback(cc, entry_no, is_urgent)) { debug_assert(clockcache_lookup(cc, addr) == entry_no); first_addr = entry->page.disk_addr; // walk backwards through extent to find first cleanable entry @@ -1023,8 +1269,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY - && clockcache_ok_to_writeback(cc, next_entry_no, is_urgent) - && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); + && clockcache_try_claim_writeback(cc, next_entry_no, is_urgent)); first_addr += page_size; end_addr = entry->page.disk_addr; // walk forwards through extent to find last cleanable entry @@ -1037,24 +1282,12 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY - && clockcache_ok_to_writeback(cc, next_entry_no, is_urgent) - && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); - + && clockcache_try_claim_writeback(cc, next_entry_no, is_urgent)); - async_io_state *state; - state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); - if (state == NULL) { - platform_error_log( - "clockcache_batch_start_writeback: async_io_state allocation " - "failed\n"); - clockcache_abort_writeback_range(cc, first_addr, end_addr); - result = STATUS_NO_MEMORY; - goto close_log; - } + async_io_state *state = clockcache_io_state_acquire(cc); - state->cc = cc; - state->outstanding_pages = NULL; - platform_status rc = io_async_state_init(state->iostate, + state->cc = cc; + platform_status rc = io_async_state_init(state->iostate, cc->io, io_async_pwritev, first_addr, @@ -1064,11 +1297,8 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) platform_error_log("clockcache_batch_start_writeback: " "io_async_state_init failed: %s\n", platform_status_to_string(rc)); - clockcache_abort_writeback_range(cc, first_addr, end_addr); - platform_free(PROCESS_PRIVATE_HEAP_ID, state); - result = rc; - goto close_log; } + platform_assert_status_ok(rc); uint64 req_count = clockcache_divide_by_page_size(cc, end_addr - first_addr); @@ -1089,12 +1319,8 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) platform_error_log("clockcache_batch_start_writeback: " "io_async_state_append_page failed: %s\n", platform_status_to_string(rc)); - io_async_state_deinit(state->iostate); - clockcache_abort_writeback_range(cc, first_addr, end_addr); - platform_free(PROCESS_PRIVATE_HEAP_ID, state); - result = rc; - goto close_log; } + platform_assert_status_ok(rc); } if (cc->cfg->use_stats) { @@ -1117,9 +1343,8 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) } } -close_log: clockcache_close_log_stream(); - return result; + return STATUS_OK; } /* @@ -1176,6 +1401,7 @@ clockcache_writeback_dirty(clockcache *cc) } // Wait for each pre-cut interval's writeback to complete, in a single pass. + platform_status result = STATUS_OK; for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { clockcache_entry *entry = clockcache_get_entry(cc, entry_no); while (TRUE) { @@ -1186,11 +1412,24 @@ clockcache_writeback_dirty(clockcache *cc) { break; } + if (clockcache_test_flag(cc, entry_no, CC_WRITEBACK_ERROR)) { + /* + * This page's write failed and it stays in CC_WRITEBACK until a + * retry succeeds, so waiting on it here would never return. Record + * the failure and keep draining the remaining entries, so that when + * we do return, no pre-cut write is still outstanding. + */ + platform_error_log("clockcache_writeback_dirty: writeback of addr " + "%lu failed\n", + entry->page.disk_addr); + result = STATUS_IO_ERROR; + break; + } clockcache_wait(cc); } } - return STATUS_OK; + return result; } /* @@ -1206,6 +1445,12 @@ clockcache_writeback_dirty(clockcache *cc) * clockcache_try_evict * * Attempts to evict the page if it is evictable + * + * Evictability requires CC_EVICTABLE_STATUS, i.e. CC_CLEAN and nothing + * else, and CC_CLEAN is set only by clockcache_dirty_complete_writeback(). + * So a page cannot leave the cache before its writeback has completed -- + * an invariant clockcache_writeback_get_page_status() relies on to treat a + * non-resident page as written. *---------------------------------------------------------------------- */ static void @@ -1280,8 +1525,14 @@ clockcache_try_evict(clockcache *cc, uint32 entry_number) /* 5. clear lookup, disk addr */ uint64 addr = entry->page.disk_addr; if (addr != CC_UNMAPPED_ADDR) { - uint64 lookup_no = clockcache_divide_by_page_size(cc, addr); - cc->lookup[lookup_no] = CC_UNMAPPED_ENTRY; + uint64 lookup_no = clockcache_divide_by_page_size(cc, addr); + bool32 unmapped = __sync_bool_compare_and_swap( + &cc->lookup[lookup_no], entry_number, CC_UNMAPPED_ENTRY); + platform_assert(unmapped, + "eviction of entry %u at addr %lu found lookup entry %u", + entry_number, + addr, + cc->lookup[lookup_no]); entry->page.disk_addr = CC_UNMAPPED_ADDR; } debug_only uint32 debug_status = @@ -1520,16 +1771,17 @@ clockcache_flush(clockcache *cc) * evicts all the pages. *----------------------------------------------------------------------------- */ -int +platform_status clockcache_evict_all(clockcache *cc, bool32 ignore_pinned_pages) { uint32 evict_hand; - uint32 i; - if (!ignore_pinned_pages) { - // there can be no references or pins or locks or it will block eviction - clockcache_assert_no_locks_held(cc); // take out for performance - } + /* + * Prefetch and writeback callbacks can leave entries temporarily loading or + * writeback-locked even after their users have dropped every reference. + * Drain those callbacks before checking the quiescent-cache invariant. + */ + io_wait_all(cc->io); // evict all the pages for (evict_hand = 0; evict_hand < cc->cfg->batch_capacity; evict_hand++) { @@ -1538,16 +1790,43 @@ clockcache_evict_all(clockcache *cc, bool32 ignore_pinned_pages) clockcache_evict_batch(cc, evict_hand); } - for (i = 0; i < cc->cfg->page_capacity; i++) { - debug_only uint32 entry_no = - clockcache_page_to_entry_number(cc, &cc->entry->page); - // Every page should either be evicted or pinned. - debug_assert( - cc->entry[i].status == CC_FREE_STATUS - || (ignore_pinned_pages && clockcache_get_pin(cc, entry_no))); + for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { + if (cc->entry[entry_no].status == CC_FREE_STATUS) { + if (cc->entry[entry_no].page.disk_addr != CC_UNMAPPED_ADDR) { + platform_error_log("clockcache_evict_all: free entry %u still " + "names addr %lu\n", + entry_no, + cc->entry[entry_no].page.disk_addr); + return STATUS_INVALID_STATE; + } + continue; + } + if (ignore_pinned_pages && clockcache_get_pin(cc, entry_no)) { + continue; + } + platform_error_log("clockcache_evict_all: entry %u at addr %lu remained " + "resident with status 0x%x\n", + entry_no, + cc->entry[entry_no].page.disk_addr, + cc->entry[entry_no].status); + return STATUS_BUSY; + } + + if (!ignore_pinned_pages) { + uint64 lookup_capacity = + allocator_get_capacity(cc->al) / clockcache_page_size(cc); + for (uint64 lookup_no = 0; lookup_no < lookup_capacity; lookup_no++) { + if (cc->lookup[lookup_no] != CC_UNMAPPED_ENTRY) { + platform_error_log("clockcache_evict_all: lookup %lu still maps " + "entry %u\n", + lookup_no, + cc->lookup[lookup_no]); + return STATUS_INVALID_STATE; + } + } } - return 0; + return STATUS_OK; } /* @@ -1570,10 +1849,13 @@ clockcache_alloc(clockcache *cc, uint64 addr, page_type type) entry->page.disk_addr = addr; entry->type = type; uint64 lookup_no = clockcache_divide_by_page_size(cc, entry->page.disk_addr); - // bool32 rc = __sync_bool_compare_and_swap( - // &cc->lookup[lookup_no], CC_UNMAPPED_ENTRY, entry_no); - // platform_assert(rc); - cc->lookup[lookup_no] = entry_no; + bool32 mapped = __sync_bool_compare_and_swap( + &cc->lookup[lookup_no], CC_UNMAPPED_ENTRY, entry_no); + platform_assert(mapped, + "allocation of entry %u at addr %lu found lookup entry %u", + entry_no, + addr, + cc->lookup[lookup_no]); clockcache_record_backtrace(cc, entry_no); clockcache_log(entry->page.disk_addr, @@ -1657,8 +1939,14 @@ clockcache_try_page_discard(clockcache *cc, uint64 addr) clockcache_get_write(cc, entry_number); /* 5. clear lookup and disk addr; set status to CC_FREE_STATUS */ - uint64 lookup_no = clockcache_divide_by_page_size(cc, addr); - cc->lookup[lookup_no] = CC_UNMAPPED_ENTRY; + uint64 lookup_no = clockcache_divide_by_page_size(cc, addr); + bool32 unmapped = __sync_bool_compare_and_swap( + &cc->lookup[lookup_no], entry_number, CC_UNMAPPED_ENTRY); + platform_assert(unmapped, + "discard of entry %u at addr %lu found lookup entry %u", + entry_number, + addr, + cc->lookup[lookup_no]); debug_assert(entry->page.disk_addr == addr); entry->page.disk_addr = CC_UNMAPPED_ADDR; @@ -2394,49 +2682,146 @@ clockcache_unpin(clockcache *cc, page_handle *page) /* *----------------------------------------------------------------------------- - * clockcache_page_writeback -- + * clockcache_writeback_claim_page -- * - * Issues writeback of the page. This does not make the page durable; it - * only hands the write to the I/O layer. + * Try to take responsibility for writing back one resident page, and say + * what stands in the way if we cannot. *gen is set for CLAIMED and + * INFLIGHT (the interval the write covers) and to 0 for NOT_NEEDED; it is + * left untouched for UNAVAILABLE. * - * With is_blocking == FALSE the writeback is issued asynchronously and - * this returns without waiting for completion. With is_blocking == TRUE - * the page is written synchronously and has completed by the time this - * returns. + * Shared by the page and extent writeback paths so that the decision -- + * and in particular the single-snapshot rule below -- lives in one place. *----------------------------------------------------------------------------- */ -void -clockcache_page_writeback(clockcache *cc, - page_handle *page, - bool32 is_blocking, - page_type type) +typedef enum clockcache_writeback_claim { + CC_WRITEBACK_CLAIMED, // we set CC_WRITEBACK; caller must issue the I/O + CC_WRITEBACK_NOT_NEEDED, // already clean: nothing to write + CC_WRITEBACK_INFLIGHT, // someone else's write already covers it + CC_WRITEBACK_UNAVAILABLE, // dirty, but locked or claimed +} clockcache_writeback_claim; + +static clockcache_writeback_claim +clockcache_writeback_claim_page(clockcache *cc, + uint32 entry_number, + uint64 *gen) { - uint32 entry_number = clockcache_page_to_entry_number(cc, page); - async_io_state *state; - uint64 addr = page->disk_addr; - const threadid tid = platform_get_tid(); - platform_status status; - - while (!clockcache_try_set_writeback(cc, entry_number, TRUE)) { - if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { - return; + while (TRUE) { + if (clockcache_try_set_writeback(cc, entry_number, TRUE)) { + /* + * Read the generation only now that CC_WRITEBACK is set. That excludes + * the writer (see clockcache_get_write), so the interval cannot + * advance under us and the receipt provably names the interval this + * write covers. + */ + *gen = clockcache_writeback_get_generation(cc, entry_number); + debug_assert(*gen != 0, "a page claimed for writeback must be dirty"); + return CC_WRITEBACK_CLAIMED; } /* - * A pressure cleaner or a checkpoint fence may have begun writeback - * after the caller released its claim. Wait for that writeback rather - * than treating a perfectly valid concurrent flush as an assertion. + * Decide from a single snapshot of the status word. Testing the flags one + * at a time lets a concurrent writeback complete in between -- dirty when + * we test CC_CLEAN, no longer in writeback when we test CC_WRITEBACK -- + * and report a failure for a page whose contents did in fact reach the + * device. */ - if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK)) { - clockcache_wait(cc); + entry_status status = clockcache_get_status(cc, entry_number); + + if (status & CC_CLEAN) { + // Includes the transient CC_CLEAN|CC_WRITEBACK state that + // clockcache_dirty_complete_writeback passes through: the write has + // completed by the time CC_CLEAN is set. + *gen = 0; + return CC_WRITEBACK_NOT_NEEDED; + } + if (status & CC_WRITEBACK_ERROR) { + /* + * An earlier write of this page failed. Our caller wants a write, + * so claim the retry and hand it back as an ordinary claim rather + * than reporting an error the caller could do nothing about. The + * generation is unchanged since the failed write, so the receipt + * still names the same interval. + * + * If another thread beat us to the retry, loop: it now holds + * CC_WRITEBACK without the error bit, so the next pass reports the + * page as in flight. + */ + if (clockcache_try_retry_writeback(cc, entry_number)) { + *gen = clockcache_writeback_get_generation(cc, entry_number); + debug_assert(*gen != 0, + "a page claimed for writeback must be dirty"); + return CC_WRITEBACK_CLAIMED; + } continue; } - - platform_assert(0, - "page_writeback requires a cleanable page: entry=%u " - "status=%u\n", + if (status & CC_WRITEBACK) { + /* + * A pressure cleaner or a checkpoint fence got here first. Report that + * in-flight interval rather than blocking on it: the caller waits on + * the receipt exactly as it would on a write we issued ourselves, and + * blocking would serialize a caller issuing writeback for many pages. + * This may read 0 if the write has completed since the snapshot, which + * correctly says there is nothing left to wait for. + */ + *gen = clockcache_writeback_get_generation(cc, entry_number); + return CC_WRITEBACK_INFLIGHT; + } + if (status & (CC_WRITELOCKED | CC_CLAIMED | CC_FREE | CC_LOADING)) { + debug_assert(!(status & (CC_FREE | CC_LOADING)), + "writeback of a free or loading page: entry=%u status=%u", entry_number, - clockcache_get_status(cc, entry_number)); + status); + return CC_WRITEBACK_UNAVAILABLE; + } + + /* + * Dirty, unlocked and not in writeback. The tests above cover every + * status bit but CC_ACCESSED, so the snapshot is CC_CLEANABLE1_STATUS or + * CC_CLEANABLE2_STATUS -- both of which try_set_writeback() accepts. The + * status therefore changed between the failed compare-and-swap and the + * snapshot, so retry rather than report a failure we already know to be + * stale. (That exhaustiveness is what keeps this from spinning on a + * status the cascade forgot to handle.) + */ + } +} + +/* + *----------------------------------------------------------------------------- + * clockcache_writeback_page -- + * + * Issues writeback of the page and fills in *req. This does not make the + * page durable; it only hands the write to the I/O layer. + * + * Never blocks. See cache_writeback_page() for the contract. + *----------------------------------------------------------------------------- + */ +platform_status +clockcache_writeback_page(clockcache *cc, + page_handle *page, + page_type type, + cache_writeback_request *req) +{ + uint32 entry_number = clockcache_page_to_entry_number(cc, page); + async_io_state *state; + uint64 addr = page->disk_addr; + const threadid tid = platform_get_tid(); + platform_status status; + + req->addr = addr; + req->gen = 0; + req->is_extent = FALSE; + + switch (clockcache_writeback_claim_page(cc, entry_number, &req->gen)) { + case CC_WRITEBACK_NOT_NEEDED: + case CC_WRITEBACK_INFLIGHT: + // Nothing for us to issue; req->gen says what to wait on, if + // anything. + return STATUS_OK; + case CC_WRITEBACK_UNAVAILABLE: + return STATUS_BUSY; + case CC_WRITEBACK_CLAIMED: + break; } if (cc->cfg->use_stats) { @@ -2444,124 +2829,114 @@ clockcache_page_writeback(clockcache *cc, cc->stats[tid].syncs_issued++; } - if (!is_blocking) { - state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); - if (state == NULL) { - platform_error_log( - "clockcache_page_writeback: async_io_state allocation " - "failed for addr %lu, entry %u, type %u\n", - addr, - entry_number, - type); - } - platform_assert(state); - state->cc = cc; - state->outstanding_pages = NULL; - status = io_async_state_init(state->iostate, - cc->io, - io_async_pwritev, - addr, - clockcache_write_callback, - state); - if (!SUCCESS(status)) { - platform_error_log( - "clockcache_page_writeback: io_async_state_init failed " - "for addr %lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); - } - platform_assert_status_ok(status); - status = io_async_state_append_page(state->iostate, page->data); - if (!SUCCESS(status)) { - platform_error_log( - "clockcache_page_writeback: io_async_state_append_page " - "failed for addr %lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); - } - platform_assert_status_ok(status); - io_async_run(state->iostate); - } else { - status = io_write(cc->io, page->data, clockcache_page_size(cc), addr); - if (!SUCCESS(status)) { - platform_error_log( - "clockcache_page_writeback: io_write failed for addr " - "%lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); - } - platform_assert_status_ok(status); - clockcache_log(addr, - entry_number, - "page_writeback write entry %u addr %lu\n", - entry_number, - addr); - clockcache_dirty_complete_writeback(cc, entry_number); + state = clockcache_io_state_acquire(cc); + state->cc = cc; + status = io_async_state_init(state->iostate, + cc->io, + io_async_pwritev, + addr, + clockcache_write_callback, + state); + if (!SUCCESS(status)) { + platform_error_log("clockcache_writeback_page: io_async_state_init " + "failed for addr %lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); + } + platform_assert_status_ok(status); + status = io_async_state_append_page(state->iostate, page->data); + if (!SUCCESS(status)) { + platform_error_log("clockcache_writeback_page: " + "io_async_state_append_page failed for addr %lu, " + "entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } + platform_assert_status_ok(status); + io_async_run(state->iostate); + return STATUS_OK; } /* *----------------------------------------------------------------------------- - * clockcache_extent_writeback -- + * clockcache_writeback_extent -- * - * Asynchronously issues writeback of the extent. This does not make the - * extent durable; it only hands the writes to the I/O layer and returns - * without waiting for completion. + * Asynchronously issues writeback of the extent and fills in *req. This + * does not make the extent durable; it only hands the writes to the I/O + * layer and returns without waiting for completion. * - * Adds the number of pages issued writeback to the counter pointed to - * by pages_outstanding. When the writes complete, a callback subtracts - * them off, so that the caller may track how many pages are in - * writeback. + * Resident dirty pages are coalesced into as few I/Os as their contiguity + * allows. Pages that are clean or not resident need no write and are + * skipped. A page that is dirty but not writeback-able fails the whole + * request: its contents will not reach the device, so reporting success + * would let a caller believe the extent had been written when it had not. * - * Assumes all pages in the extent are clean or cleanable + * req->gen is the newest dirty interval among the extent's pages, so it is + * safe to compare every page of the extent against; see + * clockcache_writeback_get_status(). *----------------------------------------------------------------------------- */ -void -clockcache_extent_writeback(clockcache *cc, - uint64 addr, - uint64 *pages_outstanding) +platform_status +clockcache_writeback_extent(clockcache *cc, + uint64 addr, + page_type type, + cache_writeback_request *req) { async_io_state *state = NULL; uint64 i; uint32 entry_number; - uint64 req_count = 0; uint64 req_addr; uint64 page_addr; + const threadid tid = platform_get_tid(); + uint64 max_gen = 0; + platform_status result = STATUS_OK; + + req->addr = addr; + req->gen = 0; + req->is_extent = TRUE; for (i = 0; i < cc->cfg->pages_per_extent; i++) { page_addr = addr + clockcache_multiply_by_page_size(cc, i); entry_number = clockcache_lookup(cc, page_addr); - if (entry_number != CC_UNMAPPED_ENTRY - && clockcache_try_set_writeback(cc, entry_number, TRUE)) - { + + /* + * A page that is not resident needs no write: eviction cannot happen + * before writeback completes (see clockcache_try_evict). + */ + uint64 gen = 0; + clockcache_writeback_claim claim = CC_WRITEBACK_NOT_NEEDED; + if (entry_number != CC_UNMAPPED_ENTRY) { + claim = clockcache_writeback_claim_page(cc, entry_number, &gen); + } + max_gen = MAX(max_gen, gen); + if (claim == CC_WRITEBACK_UNAVAILABLE) { + // No write will be issued for this page, so the extent will not be + // fully written. The caller has to hear about it. + result = STATUS_BUSY; + } + + if (claim == CC_WRITEBACK_CLAIMED) { + if (cc->cfg->use_stats) { + cc->stats[tid].page_writes[type]++; + cc->stats[tid].syncs_issued++; + } + if (state == NULL) { - req_addr = page_addr; - state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); - if (state == NULL) { - platform_error_log("clockcache_extent_writeback: async_io_state " - "allocation failed for extent addr %lu, " - "page addr %lu, entry %u\n", - addr, - page_addr, - entry_number); - } - platform_assert(state); - state->cc = cc; - state->outstanding_pages = pages_outstanding; - platform_status rc = io_async_state_init(state->iostate, + req_addr = page_addr; + state = clockcache_io_state_acquire(cc); + state->cc = cc; + platform_status rc = io_async_state_init(state->iostate, cc->io, io_async_pwritev, req_addr, clockcache_write_callback, state); if (!SUCCESS(rc)) { - platform_error_log("clockcache_extent_writeback: " + platform_error_log("clockcache_writeback_extent: " "io_async_state_init failed for extent addr " "%lu, req addr %lu, entry %u: %s\n", addr, @@ -2574,7 +2949,7 @@ clockcache_extent_writeback(clockcache *cc, platform_status rc = io_async_state_append_page( state->iostate, clockcache_get_entry(cc, entry_number)->page.data); if (!SUCCESS(rc)) { - platform_error_log("clockcache_extent_writeback: " + platform_error_log("clockcache_writeback_extent: " "io_async_state_append_page failed for extent " "addr %lu, page addr %lu, entry %u: %s\n", addr, @@ -2583,23 +2958,134 @@ clockcache_extent_writeback(clockcache *cc, platform_status_to_string(rc)); } platform_assert_status_ok(rc); - req_count++; } else { - // ALEX: There is maybe a race with eviction with this assertion - debug_assert(entry_number == CC_UNMAPPED_ENTRY - || clockcache_test_flag(cc, entry_number, CC_CLEAN)); + /* We are not writing this page, so contiguity breaks here: close out + * whatever run of pages we had accumulated. */ if (state != NULL) { - __sync_fetch_and_add(pages_outstanding, req_count); io_async_run(state->iostate); - state = NULL; - req_count = 0; + state = NULL; } } } if (state != NULL) { - __sync_fetch_and_add(pages_outstanding, req_count); io_async_run(state->iostate); } + + /* + * Publish the generation even on failure: writes were issued for the pages + * we did claim, and the caller must be able to wait them out before it + * reuses or frees the extent. + */ + req->gen = max_gen; + return result; +} + +/* + *----------------------------------------------------------------------------- + * clockcache_writeback_get_page_status -- + * + * Whether the write covering interval `gen` of the page at `addr` has + * completed. + * + * The flag is tested before the generation is read, and that order is + * load-bearing. Completion runs gen := 0, then set CC_CLEAN, then clear + * CC_WRITEBACK (see clockcache_dirty_complete_writeback), so observing + * CC_WRITEBACK clear implies the generation was already 0 at that instant. + * Reading the generation afterwards and finding it back at `gen` therefore + * proves the page was cleaned and dirtied again -- whereas reading the + * generation first would report that same pair for a write that had merely + * just completed, a false report of a concurrent writer. + *----------------------------------------------------------------------------- + */ +static cache_writeback_status +clockcache_writeback_get_page_status(clockcache *cc, uint64 addr, uint64 gen) +{ + if (gen == 0) { + // No write was issued for this page: nothing to wait for. + return CACHE_WRITEBACK_COMPLETE; + } + + uint32 entry_number = clockcache_lookup(cc, addr); + if (entry_number == CC_UNMAPPED_ENTRY) { + /* + * Not resident, so the contents reached the device before the page left + * the cache: eviction requires CC_EVICTABLE_STATUS, which is CC_CLEAN, + * and only clockcache_dirty_complete_writeback() sets CC_CLEAN. + */ + return CACHE_WRITEBACK_COMPLETE; + } + + /* + * One snapshot, so that CC_WRITEBACK and CC_WRITEBACK_ERROR are read + * consistently with each other as well as with the generation below. + */ + entry_status status = clockcache_get_status(cc, entry_number); + uint64 cur = clockcache_writeback_get_generation(cc, entry_number); + + if (clockcache_get_entry(cc, entry_number)->page.disk_addr != addr) { + // The entry was rebound under us, so it was evicted -- and hence written + // back -- and the flag and generation we just read belong to some other + // page. + return CACHE_WRITEBACK_COMPLETE; + } + if (cur == 0) { + return CACHE_WRITEBACK_COMPLETE; + } + if (cur > gen) { + // Drained, then dirtied again in a later window. + return CACHE_WRITEBACK_REDIRTIED; + } + if (status & CC_WRITEBACK_ERROR) { + // The write covering this interval failed. Reported ahead of PENDING: the + // page stays in CC_WRITEBACK until a retry succeeds, so a caller told + // PENDING here would poll forever. + return CACHE_WRITEBACK_FAILED; + } + /* + * cur <= gen: the page may still be in the interval we wrote. Note that + * cur < gen is legitimate for an extent request, whose gen is the newest + * interval among its pages -- a writeback_dirty() landing while the extent + * was being filled leaves its pages spread over two windows. + */ + return (status & CC_WRITEBACK) ? CACHE_WRITEBACK_PENDING + : CACHE_WRITEBACK_REDIRTIED; +} + +cache_writeback_status +clockcache_writeback_get_status(clockcache *cc, + const cache_writeback_request *req) +{ + uint64 num_pages = req->is_extent ? cc->cfg->pages_per_extent : 1; + bool32 redirtied = FALSE; + bool32 failed = FALSE; + + for (uint64 i = 0; i < num_pages; i++) { + uint64 addr = req->addr + clockcache_multiply_by_page_size(cc, i); + switch (clockcache_writeback_get_page_status(cc, addr, req->gen)) { + case CACHE_WRITEBACK_PENDING: + /* + * One outstanding page is enough; no need to look at the rest. Note + * this outranks a failure found earlier in the extent, which + * is what makes FAILED safe to act on: by the time the caller sees + * it, no write on the extent is still in flight. + */ + return CACHE_WRITEBACK_PENDING; + case CACHE_WRITEBACK_FAILED: + failed = TRUE; + break; + case CACHE_WRITEBACK_REDIRTIED: + // Keep going: a later page may still be pending, which the caller + // has to keep waiting for. + redirtied = TRUE; + break; + case CACHE_WRITEBACK_COMPLETE: + break; + } + } + if (failed) { + return CACHE_WRITEBACK_FAILED; + } + return redirtied ? CACHE_WRITEBACK_REDIRTIED : CACHE_WRITEBACK_COMPLETE; } /* @@ -3156,6 +3642,15 @@ clockcache_enable_sync_get(clockcache *cc, bool32 enabled) cc->per_thread[platform_get_tid()].enable_sync_get = enabled; } +static platform_status +clockcache_range_is_readable(clockcache *cc, + uint64 addr, + uint64 bytes, + bool32 *readable) +{ + return io_range_is_readable(cc->io, addr, bytes, readable); +} + static allocator * clockcache_get_allocator(const clockcache *cc) { @@ -3312,23 +3807,32 @@ clockcache_get_async_state_result_virtual(void *payload) return state->__async_result; } -void -clockcache_page_writeback_virtual(cache *c, - page_handle *page, - bool32 is_blocking, - page_type type) +platform_status +clockcache_writeback_page_virtual(cache *c, + page_handle *page, + page_type type, + cache_writeback_request *req) { clockcache *cc = (clockcache *)c; - clockcache_page_writeback(cc, page, is_blocking, type); + return clockcache_writeback_page(cc, page, type, req); } -void -clockcache_extent_writeback_virtual(cache *c, - uint64 addr, - uint64 *pages_outstanding) +platform_status +clockcache_writeback_extent_virtual(cache *c, + uint64 addr, + page_type type, + cache_writeback_request *req) +{ + clockcache *cc = (clockcache *)c; + return clockcache_writeback_extent(cc, addr, type, req); +} + +cache_writeback_status +clockcache_writeback_get_status_virtual(cache *c, + const cache_writeback_request *req) { clockcache *cc = (clockcache *)c; - clockcache_extent_writeback(cc, addr, pages_outstanding); + return clockcache_writeback_get_status(cc, req); } void @@ -3352,7 +3856,7 @@ clockcache_durable_barrier_virtual(cache *c) return io_durable_barrier(cc->io); } -int +platform_status clockcache_evict_all_virtual(cache *c, bool32 ignore_pinned) { clockcache *cc = (clockcache *)c; @@ -3450,6 +3954,15 @@ clockcache_enable_sync_get_virtual(cache *c, bool32 enabled) clockcache_enable_sync_get(cc, enabled); } +static platform_status +clockcache_range_is_readable_virtual(cache *c, + uint64 addr, + uint64 bytes, + bool32 *readable) +{ + return clockcache_range_is_readable((clockcache *)c, addr, bytes, readable); +} + allocator * clockcache_get_allocator_virtual(const cache *c) { @@ -3473,36 +3986,38 @@ static cache_ops clockcache_ops = { .page_get_async = clockcache_get_async_virtual, .page_get_async_result = clockcache_get_async_state_result_virtual, - .page_unget = clockcache_unget_virtual, - .page_try_claim = clockcache_try_claim_virtual, - .page_unclaim = clockcache_unclaim_virtual, - .page_lock = clockcache_lock_virtual, - .page_unlock = clockcache_unlock_virtual, - .page_prefetch = clockcache_prefetch_virtual, - .page_prefetch_page = clockcache_prefetch_page_virtual, - .page_pin = clockcache_pin_virtual, - .page_unpin = clockcache_unpin_virtual, - .page_writeback = clockcache_page_writeback_virtual, - .extent_writeback = clockcache_extent_writeback_virtual, - .flush = clockcache_flush_virtual, - .writeback_dirty = clockcache_writeback_dirty_virtual, - .durable_barrier = clockcache_durable_barrier_virtual, - .evict = clockcache_evict_all_virtual, - .cleanup = clockcache_wait_virtual, - .in_use = clockcache_in_use_virtual, - .assert_ungot = clockcache_assert_ungot_virtual, - .assert_free = clockcache_assert_no_locks_held_virtual, - .print = clockcache_print_virtual, - .print_stats = clockcache_print_stats_virtual, - .io_stats = clockcache_io_stats_virtual, - .reset_stats = clockcache_reset_stats_virtual, - .validate_page = clockcache_validate_page_virtual, - .count_dirty = clockcache_count_dirty_virtual, - .page_get_read_ref = clockcache_get_read_ref_virtual, - .cache_present = clockcache_present_virtual, - .enable_sync_get = clockcache_enable_sync_get_virtual, - .get_allocator = clockcache_get_allocator_virtual, - .get_config = clockcache_get_config_virtual, + .page_unget = clockcache_unget_virtual, + .page_try_claim = clockcache_try_claim_virtual, + .page_unclaim = clockcache_unclaim_virtual, + .page_lock = clockcache_lock_virtual, + .page_unlock = clockcache_unlock_virtual, + .page_prefetch = clockcache_prefetch_virtual, + .page_prefetch_page = clockcache_prefetch_page_virtual, + .page_pin = clockcache_pin_virtual, + .page_unpin = clockcache_unpin_virtual, + .page_writeback = clockcache_writeback_page_virtual, + .extent_writeback = clockcache_writeback_extent_virtual, + .writeback_get_status = clockcache_writeback_get_status_virtual, + .flush = clockcache_flush_virtual, + .writeback_dirty = clockcache_writeback_dirty_virtual, + .durable_barrier = clockcache_durable_barrier_virtual, + .evict = clockcache_evict_all_virtual, + .cleanup = clockcache_wait_virtual, + .in_use = clockcache_in_use_virtual, + .assert_ungot = clockcache_assert_ungot_virtual, + .assert_free = clockcache_assert_no_locks_held_virtual, + .print = clockcache_print_virtual, + .print_stats = clockcache_print_stats_virtual, + .io_stats = clockcache_io_stats_virtual, + .reset_stats = clockcache_reset_stats_virtual, + .validate_page = clockcache_validate_page_virtual, + .count_dirty = clockcache_count_dirty_virtual, + .page_get_read_ref = clockcache_get_read_ref_virtual, + .cache_present = clockcache_present_virtual, + .enable_sync_get = clockcache_enable_sync_get_virtual, + .range_is_readable = clockcache_range_is_readable_virtual, + .get_allocator = clockcache_get_allocator_virtual, + .get_config = clockcache_get_config_virtual, }; /* @@ -3583,6 +4098,22 @@ clockcache_init(clockcache *cc, // OUT // at 1. cc->dirty_generation = 1; + /* + * Allocated from the process-private heap, matching the per-I/O states it + * substitutes for: the io layer's contexts are process-local. + */ + cc->io_state_pool = TYPED_ARRAY_MALLOC( + PROCESS_PRIVATE_HEAP_ID, cc->io_state_pool, CC_IO_STATE_POOL_SIZE); + if (cc->io_state_pool == NULL) { + platform_error_log("clockcache_init: failed to allocate the io state " + "reserve (%lu bytes)\n", + CC_IO_STATE_POOL_SIZE * sizeof(cc->io_state_pool[0])); + goto alloc_error; + } + cc->io_state_pool_free = (CC_IO_STATE_POOL_SIZE == 64) + ? ~0ULL + : ((1ULL << CC_IO_STATE_POOL_SIZE) - 1); + /* lookup maps addrs to entries, entry contains the entries themselves */ platform_status rc = platform_buffer_init( &cc->lookup_bh, allocator_page_capacity * sizeof(cc->lookup[0])); @@ -3767,4 +4298,8 @@ clockcache_deinit(clockcache *cc) // IN/OUT } cc->batch_busy = NULL; } + if (cc->io_state_pool) { + platform_free(PROCESS_PRIVATE_HEAP_ID, cc->io_state_pool); + cc->io_state_pool = NULL; + } } diff --git a/src/clockcache.h b/src/clockcache.h index cb5929fd..cbe790ce 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -45,6 +45,7 @@ typedef struct clockcache_config { typedef struct clockcache clockcache; typedef struct clockcache_entry clockcache_entry; +typedef struct async_io_state async_io_state; #ifdef RECORD_ACQUISITION_STACKS @@ -149,6 +150,16 @@ struct clockcache { // below that cut. uint64 dirty_generation; + /* + * A reserve of async I/O states, so that a writeback can always be issued + * even when the heap is exhausted -- which matters because writing pages + * back is how memory pressure gets relieved in the first place. Drawn from + * only when a heap allocation fails; see clockcache_io_state_acquire(). + * io_state_pool_free is a bitvector, one bit per slot, set when free. + */ + async_io_state *io_state_pool; + volatile uint64 io_state_pool_free; + volatile struct { volatile uint32 free_hand; bool32 enable_sync_get; diff --git a/src/core.c b/src/core.c index 84134b93..ad0e3cfb 100644 --- a/src/core.c +++ b/src/core.c @@ -182,12 +182,75 @@ core_checkpoint_capture_cut(core_handle *spl, static superblock_log_head core_log_to_superblock_log_head(log_head info, uint64 start_generation) { - return (superblock_log_head){.addr = info.addr, - .meta_addr = info.meta_addr, - .magic = info.magic, + return (superblock_log_head){.head = info, .start_generation = start_generation}; } +/* Does the durable record name this concrete log stream as its live log? */ +static bool32 +core_superblock_log_head_matches(superblock_log_head recorded, log_head live) +{ + return !SUPERBLOCK_NO_LOG(recorded) + && log_head_is_equal(recorded.head, live); +} + +/* + * Superblock transitions are staged directly in the context's in-memory + * image. Keep an explicit before-image around any transition that can be + * retried while the core remains mounted. In particular, + * superblock_make_durable() increments the generation before doing I/O, so a + * failed write/barrier must not leave the semantic transition staged for a + * caller that will apply it again. + */ +static void +core_superblock_save_image(core_handle *spl, superblock *saved) +{ + memcpy(saved, spl->superblock.image, sizeof(*saved)); +} + +static void +core_superblock_restore_image(core_handle *spl, const superblock *saved) +{ + memcpy(spl->superblock.image, saved, sizeof(*saved)); +} + +/* + * Is the live trunk cut already exactly the one in the confirmed superblock + * image? This recognizes a checkpoint that made all data durable before an + * unmount's redundant publication encounters an I/O error. + */ +static bool32 +core_current_root_matches_durable_record(core_handle *spl) +{ + trunk_snapshot snapshot; + uint64 first_unincorporated_generation; + platform_status rc = core_checkpoint_capture_cut( + spl, &snapshot, &first_unincorporated_generation); + if (!SUCCESS(rc)) { + platform_error_log("core_unmount: could not inspect the current root: " + "%s\n", + platform_status_to_string(rc)); + return FALSE; + } + + superblock_tree_record rec; + superblock_get_tree_record(&spl->superblock, &rec); + bool32 matches = + snapshot.root_addr == rec.root_addr + && first_unincorporated_generation == rec.first_unincorporated_generation; + + uint64 snapshot_addr = snapshot.root_addr; + rc = trunk_snapshot_release(&spl->trunk_context, &snapshot); + if (!SUCCESS(rc)) { + platform_error_log("core_unmount: failed to release the root snapshot " + "used for the durable-root check at addr %lu: %s\n", + snapshot_addr, + platform_status_to_string(rc)); + spl->allocator_map_needs_rebuild = TRUE; + } + return matches; +} + /* * Commit the trunk's current COW root as the new durable tree root: capture the * root, make its pages durable, snapshot it into the superblock (recording the @@ -203,7 +266,11 @@ core_log_to_superblock_log_head(log_head info, uint64 start_generation) * durability checkpoint, and unmount (Part A). On success the captured * reference becomes the durable record's and the previously published root's is * released; republishing an unchanged root is just the degenerate case of that, - * so it needs no special handling. + * so it needs no special handling. Failure to release the old root after the + * publication is reported and makes the allocator map non-persistable, but it + * does not turn a successful durable publication into a checkpoint failure: + * the return value reports publication, while the sticky allocator bit reports + * post-publication reference-cleanup trouble. * * Note this always publishes, even when the root is unchanged: callers stage * log transitions into the image beforehand, and the generation bound can @@ -218,6 +285,7 @@ core_checkpoint_commit_current_root(core_handle *spl) uint64 first_unincorporated_generation; uint64 old_root_addr = 0; superblock_tree_record old_rec; + superblock saved_superblock; /* * The snapshot cut, durable record write, and old-root release are one @@ -253,6 +321,7 @@ core_checkpoint_commit_current_root(core_handle *spl) // The previously published root, retained until the new one is durable. superblock_get_tree_record(&spl->superblock, &old_rec); old_root_addr = old_rec.root_addr; + core_superblock_save_image(spl, &saved_superblock); /* * Snapshot the new root and make it durable. snapshot_tree invalidates the @@ -264,8 +333,18 @@ core_checkpoint_commit_current_root(core_handle *spl) rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { - /* The old root is still the newest durable one; keep its reference. */ - goto release_snapshot; + /* + * Restore the last confirmed image so a retry does not reapply the root + * transition to its own staged result. The write/barrier failure has an + * ambiguous outcome, however: the candidate slot may have reached disk. + * Retain the candidate root's snapshot reference conservatively so that + * a crash cannot find a durable record pointing at a root we later + * recycled. The resulting possible overcount is repaired by recovery. + */ + core_superblock_restore_image(spl, &saved_superblock); + snapshot.root_addr = 0; + spl->allocator_map_needs_rebuild = TRUE; + goto unlock_superblock; } /* @@ -289,9 +368,7 @@ core_checkpoint_commit_current_root(core_handle *spl) "%lu: %s\n", old_root_addr, platform_status_to_string(release_rc)); - if (SUCCESS(rc)) { - rc = release_rc; - } + spl->allocator_map_needs_rebuild = TRUE; } } @@ -299,19 +376,25 @@ core_checkpoint_commit_current_root(core_handle *spl) release_snapshot: { + uint64 snapshot_addr = snapshot.root_addr; platform_status release_rc = trunk_snapshot_release(&spl->trunk_context, &snapshot); - if (SUCCESS(rc) && !SUCCESS(release_rc)) { - rc = release_rc; + if (!SUCCESS(release_rc)) { + platform_error_log("core_checkpoint_commit_current_root: failed to " + "release unpublished root snapshot at addr %lu: %s\n", + snapshot_addr, + platform_status_to_string(release_rc)); + spl->allocator_map_needs_rebuild = TRUE; + if (SUCCESS(rc)) { + rc = release_rc; + } } } unlock_superblock: { platform_status unlock_rc = platform_mutex_unlock(&spl->superblock_lock); - if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { - rc = unlock_rc; - } + platform_assert_status_ok(unlock_rc); } return rc; } @@ -321,14 +404,15 @@ core_checkpoint_commit_current_root(core_handle *spl) * Incorporation-driven checkpoint (two-log protocol) * * A checkpoint rotates the log and advances the durable root without stopping - * the world. It is driven off memtable rotation and incorporation: + * the world. Event-specific functions report facts to the state machine: * - * begin (rotation): pre-create the next live log outside the critical - * section, swap it in under the insert lock (so no writer can be - * mid-write to the old log), then seal the old log just after. - * complete (incorporation): once the sealed log's generations are folded - * into the trunk root, publish the advanced root with the sealed - * slot cleared and free the sealed log's extents. + * - core_checkpoint_request() pre-creates the next live log outside the + * rotation critical section and arms the swap. + * - core_checkpoint_rotated_locked() swaps it in while writers are excluded + * from the old log. + * - core_checkpoint_advance() seals and publishes the cut, or, once + * incorporation has folded the cut generation into the trunk, publishes the + * advanced root and frees the retired log. * * See core_checkpoint_state in core.h for the phase machine and its locking. *----------------------------------------------------------------------------- @@ -340,13 +424,15 @@ core_checkpoint_commit_current_root(core_handle *spl) * the only measure that tracks a workload which overwrites in place, filling * the log without ever filling a memtable. * - * Callers hold checkpoint_state_lock, which is also held while spl->log is + * The caller holds checkpoint_state_lock, which is also held while spl->log is * swapped, so the log read below cannot race the cut. */ static bool32 core_should_take_checkpoint(core_handle *spl) { - if (!spl->cfg.use_log || spl->cfg.checkpoint_log_size_bytes == 0) { + if (!spl->cfg.use_log || spl->log == NULL + || spl->cfg.checkpoint_log_size_bytes == 0) + { return FALSE; } /* @@ -357,250 +443,339 @@ core_should_take_checkpoint(core_handle *spl) } /* - * A consistent read of what a waiter needs to poll: the current phase (has the - * cut happened yet?) and the completion count (has a given checkpoint finished - * and freed its log?). Taken together under one acquisition of the state lock. + * Every live checkpoint-state transition and observation enters through one of + * the event-specific functions below. Their arguments describe the fact being + * reported directly rather than wrapping it in a generic event structure. + * + * core_checkpoint_rotated_locked() is the sole function legal under the + * exclusive memtable insert lock and is deliberately I/O-free. All others are + * called without memtable, checkpoint-state, or superblock locks held. + * core_checkpoint_cleanup_quiesced() is destructive and is legal only after + * task/API quiescence. */ -typedef struct core_checkpoint_status { - core_checkpoint_phase phase; - uint64 completions; -} core_checkpoint_status; +typedef enum core_checkpoint_request_mode { + CORE_CHECKPOINT_REQUEST_IF_DUE = 0, + CORE_CHECKPOINT_REQUEST_REQUIRED, +} core_checkpoint_request_mode; -static core_checkpoint_status -core_checkpoint_status_get(core_handle *spl) +/* + * Results expose semantic predicates rather than the raw phase machine. A + * ticket is complete only after its retired log has been released. + */ +typedef struct core_checkpoint_result { + /* Request returns its ticket; observe echoes the ticket it was given. */ + uint64 ticket; + + /* Identity of the PENDING incarnation observed by this result. */ + uint64 pending_epoch; + + /* Predicates for `ticket`, valid for every function's result. */ + bool32 ticket_complete; + bool32 ticket_needs_rearm; + bool32 rotation_pending; + + /* An automatic PENDING checkpoint has consumed its byte grace period. */ + bool32 automatic_rotation_force_due; + + /* Quiesced shutdown view of an unpublished, already-sealed cut. */ + bool32 unpublished_sealed_log; + log_head retiring_log; +} core_checkpoint_result; + +/* + * Fill the common semantic view returned by every checkpoint state-machine + * function. The operation-specific function supplies the ticket it wants + * interpreted; the remaining predicates are sampled together under the state + * lock. + */ +static void +core_checkpoint_fill_result(core_handle *spl, + uint64 ticket, + core_checkpoint_result *result) { + if (result == NULL) { + return; + } + + ZERO_CONTENTS(result); + result->ticket = ticket; + platform_mutex_lock(&spl->checkpoint_state_lock); - core_checkpoint_status status = {.phase = spl->checkpoint.phase, - .completions = spl->checkpoint.completions}; + result->pending_epoch = spl->checkpoint.phase == CORE_CHECKPOINT_PENDING + ? spl->checkpoint.pending_epoch + : 0; + result->ticket_complete = + ticket == 0 || spl->checkpoint.completions >= ticket; + result->ticket_needs_rearm = + ticket != 0 && !result->ticket_complete + && spl->checkpoint.phase == CORE_CHECKPOINT_IDLE; + result->rotation_pending = + ticket != 0 && !result->ticket_complete + && spl->checkpoint.phase == CORE_CHECKPOINT_PENDING + && spl->checkpoint.completions + 1 == ticket; + result->automatic_rotation_force_due = + result->rotation_pending + && spl->checkpoint.force_at_log_size != UINT64_MAX + && log_get_size(spl->log) >= spl->checkpoint.force_at_log_size; + result->unpublished_sealed_log = + spl->checkpoint.phase == CORE_CHECKPOINT_SEALING + && spl->checkpoint.log_to_seal == NULL; + result->retiring_log = spl->checkpoint.sealed_head; platform_mutex_unlock(&spl->checkpoint_state_lock); - return status; } /* - * Begin, step 1 (outside the rotation critical section): if no checkpoint is in - * progress, and either the caller forces it or the interval policy says so, - * pre-create the next live log and arm the swap. Log creation does no disk - * I/O, but is kept off the insert-blocking path. - * - * `force` bypasses only the interval policy, never the use_log precondition: - * without a log there is nothing to cut, and arming would leave the rotate hook - * dereferencing a NULL spl->log. - * - * Returns a completion ticket for the checkpoint this call armed: it has - * finished, and freed its retired log, once checkpoint.completions reaches the - * ticket. The ticket is captured under the same lock acquisition that arms, so - * it cannot miss or over-count a completion. Returns 0 if this call did not - * arm one, which is not an error -- only one checkpoint can be in flight at a - * time, and declining is the normal outcome when one already is. Tickets are - * 1-based, so 0 is unambiguous. + * Saturating addition for byte boundaries. UINT64_MAX means the boundary can + * never be reached; unlike ordinary wraparound, that safely disables the + * forced-rotation backstop for an unusually large configured grace interval. */ -static uint64 -core_checkpoint_begin(core_handle *spl, bool32 force) +static inline uint64 +core_saturating_add(uint64 lhs, uint64 rhs) { - if (!spl->cfg.use_log) { - return 0; + return rhs > UINT64_MAX - lhs ? UINT64_MAX : lhs + rhs; +} + +/* + * Publish a previously sealed log cut. core_checkpoint_advance() owns the + * phase transition around this effect; this helper touches only the serialized + * superblock image. + */ +static platform_status +core_checkpoint_publish_log_cut(core_handle *spl, superblock_log_head live) +{ + platform_status rc = platform_mutex_lock(&spl->superblock_lock); + if (!SUCCESS(rc)) { + return rc; } + superblock saved_superblock; + core_superblock_save_image(spl, &saved_superblock); + /* The current image still names the retiring stream as live. */ + superblock_log_cut(&spl->superblock, live); + rc = superblock_make_durable(&spl->superblock); + if (!SUCCESS(rc)) { + /* A retry must start from the last confirmed image. */ + core_superblock_restore_image(spl, &saved_superblock); + } + + platform_status unlock_rc = platform_mutex_unlock(&spl->superblock_lock); + platform_assert_status_ok(unlock_rc); + return rc; +} + +/* + * Arm a checkpoint, either because the size policy says one is due or because + * a synchronous caller requires one. Allocation happens outside the state + * lock and is revalidated before the pending log is installed. + */ +static platform_status +core_checkpoint_request(core_handle *spl, + core_checkpoint_request_mode mode, + uint64 expected_ticket, + core_checkpoint_result *result) +{ + platform_assert(mode == CORE_CHECKPOINT_REQUEST_IF_DUE + || mode == CORE_CHECKPOINT_REQUEST_REQUIRED); + const bool32 required = mode == CORE_CHECKPOINT_REQUEST_REQUIRED; + platform_assert(expected_ticket == 0 || required); + + platform_status rc = STATUS_OK; + uint64 ticket = 0; + bool32 create_log = FALSE; + platform_mutex_lock(&spl->checkpoint_state_lock); - bool32 begin = spl->checkpoint.phase == CORE_CHECKPOINT_IDLE - && (force || core_should_take_checkpoint(spl)); + if (!spl->cfg.use_log || spl->log == NULL) { + platform_mutex_unlock(&spl->checkpoint_state_lock); + goto out; + } + if (expected_ticket != 0 && spl->checkpoint.completions >= expected_ticket) { + ticket = expected_ticket; + platform_mutex_unlock(&spl->checkpoint_state_lock); + goto out; + } + if (spl->checkpoint.phase != CORE_CHECKPOINT_IDLE) { + ticket = spl->checkpoint.completions + 1; + platform_mutex_unlock(&spl->checkpoint_state_lock); + goto out; + } + create_log = required || core_should_take_checkpoint(spl); platform_mutex_unlock(&spl->checkpoint_state_lock); - if (!begin) { - return 0; + if (!create_log) { + goto out; } - log_handle *next = shard_log_create( - spl->cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); - if (next == NULL) { - platform_error_log( - "core_checkpoint_begin: shard_log_create failed; skipping\n"); - return 0; + /* + * Allocate outside the state lock, then revalidate. A racing request + * may install its log while this allocation is in progress. + */ + log_handle *next; + rc = shard_log_create( + spl->cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id, &next); + if (!SUCCESS(rc)) { + goto out; } log_head next_head = log_get_head(next); - uint64 ticket = 0; platform_mutex_lock(&spl->checkpoint_state_lock); - if (spl->checkpoint.phase == CORE_CHECKPOINT_IDLE) { + if (expected_ticket != 0 && spl->checkpoint.completions >= expected_ticket) { + ticket = expected_ticket; + } else if (spl->checkpoint.phase == CORE_CHECKPOINT_IDLE + && (required || core_should_take_checkpoint(spl))) + { + uint64 force_at_log_size = UINT64_MAX; + if (!required && spl->cfg.checkpoint_log_grace_bytes != UINT64_MAX) { + force_at_log_size = core_saturating_add( + log_get_size(spl->log), spl->cfg.checkpoint_log_grace_bytes); + } spl->checkpoint.pending_log = next; - spl->checkpoint.live_head = next_head; - spl->checkpoint.phase = CORE_CHECKPOINT_PENDING; - next = NULL; // handed off to the checkpoint - // Ours is the next completion to be counted. + spl->checkpoint.pending_epoch++; + platform_assert(spl->checkpoint.pending_epoch != 0); + spl->checkpoint.force_at_log_size = force_at_log_size; + spl->checkpoint.phase = CORE_CHECKPOINT_PENDING; + next = NULL; // owned by checkpoint state + ticket = spl->checkpoint.completions + 1; + } else if (spl->checkpoint.phase != CORE_CHECKPOINT_IDLE) { ticket = spl->checkpoint.completions + 1; } platform_mutex_unlock(&spl->checkpoint_state_lock); + /* We lost a race to start the next checkpoint, so our speculatively created + * log is not needed. */ if (next != NULL) { - // Lost a race with a concurrent rotation; discard the speculative log. - log_seal(next); - log_dec_ref(spl->cc, &next_head); + log_deinit(next); + shard_log_dec_ref(spl->cc, &next_head); } - return ticket; -} - -/* The automatic, policy-driven arm, run after every rotation. */ -static void -core_checkpoint_maybe_begin(core_handle *spl) -{ - core_checkpoint_begin(spl, FALSE /* force */); -} -/* - * Act on the size policy from the insert path. Called by core_insert() once - * the insert lock is released. - * - * A rotation is the only point at which the log can be cut, and a workload that - * overwrites in place updates the memtable without growing it -- so it may - * never fill a memtable, never rotate, and never give - * core_checkpoint_maybe_begin() (which only runs after a rotation) a chance to - * arm anything. Left to itself the log would grow without bound. - * - * Arming and then forcing the rotation cuts the log in a single rotation, since - * the rotate hook finds the checkpoint already PENDING. Only the thread that - * actually armed goes on to force, so concurrent inserters do not pile on. - * - * Not forcing the arm below is what makes this safe against a stale flag. The - * flag is only a hint -- sampled on some earlier insert, and readable by - * several threads at once -- so a thread can arrive here long after the log it - * observed was already cut. Passing force = FALSE has core_checkpoint_begin() - * re-check the policy under the state lock against the *current* log, which - * declines in exactly those cases (the fresh log reports zero bytes, or a - * checkpoint is still in flight) and proceeds only when another cut is - * genuinely due. Forcing here would instead cut again on a log that no longer - * needs it. - */ -static void -core_maybe_cut_oversized_log(core_handle *spl) -{ - if (!spl->log_reached_threshold) { - return; - } - if (core_checkpoint_begin(spl, FALSE /* force */) != 0) { - memtable_force_rotation(&spl->mt_ctxt); - } +out: + core_checkpoint_fill_result(spl, ticket, result); + return rc; } /* - * Begin, step 2 (inside the rotation critical section, insert lock held - * exclusively): swap the pre-created live log in. Registered as - * mt_ctxt.rotate. Every log writer holds the insert lock shared across its - * log_write, so once this store retires no writer is mid-write to, or will - * newly enter, the old log -- making the subsequent seal safe. + * Report a memtable rotation while the caller holds insert exclusion. This is + * the sole checkpoint function that swaps spl->log and it remains I/O-free. */ -static void -core_rotate_log(void *arg, uint64 finalized_generation) +static platform_status +core_checkpoint_rotated_locked(core_handle *spl, + uint64 finalized_generation, + core_checkpoint_result *result) { - core_handle *spl = arg; - platform_mutex_lock(&spl->checkpoint_state_lock); if (spl->checkpoint.phase == CORE_CHECKPOINT_PENDING) { - spl->checkpoint.log_to_seal = spl->log; - spl->checkpoint.sealed_head = log_get_head(spl->log); - spl->log = spl->checkpoint.pending_log; - spl->checkpoint.pending_log = NULL; - /* - * The retiring log received everything up to and including - * finalized_generation, so the new live log's coverage starts at the next - * one. The retiring log's own start generation needs no tracking here: - * the superblock already records it, and superblock_log_cut() carries it - * across into the sealed slot. - */ + platform_assert(spl->log != NULL); + platform_assert(spl->checkpoint.pending_log != NULL); + spl->checkpoint.log_to_seal = spl->log; + spl->checkpoint.sealed_head = log_get_head(spl->log); + spl->log = spl->checkpoint.pending_log; + spl->checkpoint.pending_log = NULL; + spl->checkpoint.force_at_log_size = 0; spl->checkpoint.live_start_generation = finalized_generation + 1; spl->checkpoint.cut_generation = finalized_generation; spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; - /* - * The size hint described the log we just retired; the fresh one has had - * nothing appended. This is the only place the hint is cleared, and the - * insert lock is held exclusively here, so it cannot race the stores in - * core_log_insert(). A rotation that does not cut leaves the hint alone, - * which is correct: the same log is still live and still oversized. - */ - spl->log_reached_threshold = FALSE; + /* The size hint belonged to the stream just retired. */ + __atomic_store_n(&spl->log_reached_threshold, FALSE, __ATOMIC_RELAXED); } platform_mutex_unlock(&spl->checkpoint_state_lock); + + core_checkpoint_fill_result(spl, 0, result); + return STATUS_OK; } /* - * Begin, step 3 (just after the rotation critical section): seal the - * swapped-out log and publish the cut. The swap drained and excluded all - * writers, so sealing is safe. - * - * Publishing here is what makes the cut crash-safe. The rotation moved inserts - * to the new live log, but the superblock still names the old one, so until - * this runs a crash would lose everything written to the new log. Order - * matters: the sealed log's pages must be durable before the superblock names - * it as sealed, since recovery replays it as-is. + * Advance any eligible checkpoint work. Long-running effects use + * claim/run/settle: claim PUBLISHING or COMPLETING under the state lock, + * release it for log/superblock work, then settle the exact claimed phase. + * There is intentionally no state-machine-wide mutex because forced rotation + * and task execution can synchronously re-enter through rotation and + * incorporation. */ -static void -core_checkpoint_seal_cut(core_handle *spl) +static platform_status +core_checkpoint_advance(core_handle *spl, core_checkpoint_result *result) { - platform_mutex_lock(&spl->checkpoint_state_lock); + platform_status rc = STATUS_OK; + + /* Claim and, if necessary, seal and publish the cut. */ log_handle *to_seal = NULL; superblock_log_head live = {0}; + bool32 publish = FALSE; + + platform_mutex_lock(&spl->checkpoint_state_lock); if (spl->checkpoint.phase == CORE_CHECKPOINT_SEALING) { - to_seal = spl->checkpoint.log_to_seal; - spl->checkpoint.log_to_seal = NULL; - spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; - live = core_log_to_superblock_log_head( - spl->checkpoint.live_head, spl->checkpoint.live_start_generation); + platform_assert(spl->log != NULL); + publish = TRUE; + to_seal = spl->checkpoint.log_to_seal; + spl->checkpoint.phase = CORE_CHECKPOINT_PUBLISHING; + live = core_log_to_superblock_log_head( + log_get_head(spl->log), spl->checkpoint.live_start_generation); } platform_mutex_unlock(&spl->checkpoint_state_lock); - if (to_seal == NULL) { - return; + if (publish) { + if (to_seal != NULL) { + /* + * Sealing makes the retired log and its referenced blobs + * durable. The cut needs no cache-wide writeback: replay walks + * the stream's page links, and allocator recovery rebuilds its + * state by walking the durable tree and logs. + */ + rc = log_seal(to_seal); + if (!SUCCESS(rc)) { + platform_mutex_lock(&spl->checkpoint_state_lock); + platform_assert(spl->checkpoint.phase + == CORE_CHECKPOINT_PUBLISHING); + platform_assert(spl->checkpoint.log_to_seal == to_seal); + spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; + platform_mutex_unlock(&spl->checkpoint_state_lock); + platform_error_log( + "core_checkpoint_advance: failed to seal the log; leaving " + "the cut unpublished to retry: %s\n", + platform_status_to_string(rc)); + goto out; + } + + /* + * A successful seal consumes the handle even if publication + * later fails; a retry must publish without sealing twice. + */ + log_deinit(to_seal); + platform_mutex_lock(&spl->checkpoint_state_lock); + platform_assert(spl->checkpoint.phase == CORE_CHECKPOINT_PUBLISHING); + platform_assert(spl->checkpoint.log_to_seal == to_seal); + spl->checkpoint.log_to_seal = NULL; + platform_mutex_unlock(&spl->checkpoint_state_lock); + } + + rc = core_checkpoint_publish_log_cut(spl, live); + platform_mutex_lock(&spl->checkpoint_state_lock); + platform_assert(spl->checkpoint.phase == CORE_CHECKPOINT_PUBLISHING); + if (SUCCESS(rc)) { + spl->checkpoint.publications++; + platform_assert(spl->checkpoint.publications != 0); + } + spl->checkpoint.phase = + SUCCESS(rc) ? CORE_CHECKPOINT_INCORPORATING : CORE_CHECKPOINT_SEALING; + platform_mutex_unlock(&spl->checkpoint_state_lock); + if (!SUCCESS(rc)) { + platform_error_log( + "core_checkpoint_advance: failed to publish the log cut; " + "will retry: %s\n", + platform_status_to_string(rc)); + goto out; + } } - log_seal(to_seal); /* - * Serialize against any other superblock publisher (the superblock context - * is not thread safe). On failure the checkpoint still proceeds: the - * completion publish will record the correct final state; only this - * crash-protection window is left uncovered. + * Re-evaluate completion even after publishing the cut in this same + * invocation. Incorporation may have finished while PUBLISHING was + * owned, and that edge will not necessarily be delivered again. */ - platform_status rc = platform_mutex_lock(&spl->superblock_lock); - if (!SUCCESS(rc)) { - platform_error_log("core_checkpoint_seal_cut: lock failed: %s\n", - platform_status_to_string(rc)); - return; - } - rc = cache_writeback_dirty(spl->cc); - if (SUCCESS(rc)) { - rc = cache_durable_barrier(spl->cc); - } - if (SUCCESS(rc)) { - // The image still names the retiring log as live, so the cut moves it - // into the sealed slot, carrying its recorded start generation with it. - superblock_log_cut(&spl->superblock, live); - rc = superblock_make_durable(&spl->superblock); - } - if (!SUCCESS(rc)) { - platform_error_log("core_checkpoint_seal_cut: failed to publish the log " - "cut: %s\n", - platform_status_to_string(rc)); - } - platform_status unlock_rc = platform_mutex_unlock(&spl->superblock_lock); - platform_assert_status_ok(unlock_rc); -} - -/* - * Complete (after an incorporation): if the sealed log's generations are all - * folded into the trunk root, advance the durable root with the sealed slot - * cleared (which makes the root and superblock durable) and free the sealed - * log's extents. Called from the single-threaded incorporation path. - */ -static platform_status -core_maybe_complete_checkpoint(core_handle *spl) -{ + log_head sealed = {0}; + bool32 complete = FALSE; platform_mutex_lock(&spl->checkpoint_state_lock); - // The sealed log's cut generation is fully incorporated once it falls below - // the first unincorporated generation. When nothing has been retired, - // memtable_generation_retired() is UINT64_MAX and this wraps to 0, so the - // comparison is false without a sentinel check. uint64 first_unincorporated = memtable_generation_retired(&spl->mt_ctxt) + 1; - bool32 complete = spl->checkpoint.phase == CORE_CHECKPOINT_INCORPORATING - && first_unincorporated > spl->checkpoint.cut_generation; - log_head sealed = {0}; + complete = spl->checkpoint.phase == CORE_CHECKPOINT_INCORPORATING + && first_unincorporated > spl->checkpoint.cut_generation; if (complete) { sealed = spl->checkpoint.sealed_head; spl->checkpoint.phase = CORE_CHECKPOINT_COMPLETING; @@ -608,79 +783,337 @@ core_maybe_complete_checkpoint(core_handle *spl) platform_mutex_unlock(&spl->checkpoint_state_lock); if (!complete) { - return STATUS_OK; + goto out; } - /* - * The live log is already recorded (core_checkpoint_seal_cut() published the - * cut), so this only advances the root -- which is what lets snapshot_tree - * drop the now-covered sealed log. - */ - platform_status rc = core_checkpoint_commit_current_root(spl); + rc = core_checkpoint_commit_current_root(spl); if (SUCCESS(rc)) { - // The sealed log's entries are now durably in the root; free its extents. - log_dec_ref(spl->cc, &sealed); + /* Completion is not observable until reclamation has happened. */ + shard_log_dec_ref(spl->cc, &sealed); } else { - platform_error_log("core_maybe_complete_checkpoint: publish failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_checkpoint_advance: completion publish failed: %s\n", + platform_status_to_string(rc)); } platform_mutex_lock(&spl->checkpoint_state_lock); + platform_assert(spl->checkpoint.phase == CORE_CHECKPOINT_COMPLETING); if (SUCCESS(rc)) { ZERO_CONTENTS(&spl->checkpoint.sealed_head); - ZERO_CONTENTS(&spl->checkpoint.live_head); spl->checkpoint.cut_generation = 0; - /* - * Count the completion only here, after log_dec_ref() above: waiters take - * this as proof the retired log's space is back. - */ spl->checkpoint.completions++; spl->checkpoint.phase = CORE_CHECKPOINT_IDLE; } else { - // Leave the sealed slot intact and retry on a later incorporation. spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; } platform_mutex_unlock(&spl->checkpoint_state_lock); - // Outside the lock: this is a per-thread counter, so it needs none. if (SUCCESS(rc) && spl->cfg.use_stats) { spl->stats[platform_get_tid()].checkpoints_completed++; } + +out: + core_checkpoint_fill_result(spl, 0, result); return rc; } /* - * Release any resources of an in-flight checkpoint during a quiesced shutdown, - * before the unmount/destroy publish. No locking: the caller has quiesced all - * inserts and incorporations. A completed checkpoint - * (INCORPORATING/COMPLETING) is normally already reaped by the quiesce drain; - * the residual cases below are defensive. + * Wait until the checkpoint cut identified by `target` is durably published. + * Help a SEALING retry ourselves; if another thread owns PUBLISHING, poll + * until it settles. Incorporation and completion are deliberately outside + * this wait -- once the cut is published, either that sealed/live pair or a + * later completed root is already a complete recovery route. */ -static void -core_checkpoint_cleanup_for_shutdown(core_handle *spl) +static platform_status +core_checkpoint_wait_for_publication(core_handle *spl, uint64 target) { + uint64 wait = 100; + while (TRUE) { + platform_mutex_lock(&spl->checkpoint_state_lock); + bool32 published = spl->checkpoint.publications >= target; + platform_mutex_unlock(&spl->checkpoint_state_lock); + if (published) { + return STATUS_OK; + } + + platform_status rc = core_checkpoint_advance(spl, NULL); + + /* + * advance() can publish the cut and then encounter an unrelated root- + * completion error in the same call. The barrier only needs the former, + * so publication wins over that later error. + */ + platform_mutex_lock(&spl->checkpoint_state_lock); + published = spl->checkpoint.publications >= target; + platform_mutex_unlock(&spl->checkpoint_state_lock); + if (published) { + return STATUS_OK; + } + if (!SUCCESS(rc)) { + return rc; + } + + task_perform_one_if_needed(spl->ts, 0); + platform_sleep_ns(wait); + wait = wait > 2048 ? wait : 2 * wait; + } +} + +/* Cancel the still-pending checkpoint identified by ticket and epoch. */ +static platform_status +core_checkpoint_cancel_pending(core_handle *spl, + uint64 ticket, + uint64 pending_epoch, + core_checkpoint_result *result) +{ + log_handle *pending = NULL; + log_head pending_head = {0}; + + platform_mutex_lock(&spl->checkpoint_state_lock); + if (ticket != 0 && spl->checkpoint.phase == CORE_CHECKPOINT_PENDING + && spl->checkpoint.completions + 1 == ticket + && spl->checkpoint.pending_epoch == pending_epoch) + { + pending = spl->checkpoint.pending_log; + spl->checkpoint.pending_log = NULL; + spl->checkpoint.force_at_log_size = 0; + spl->checkpoint.phase = CORE_CHECKPOINT_IDLE; + platform_assert(pending != NULL); + pending_head = log_get_head(pending); + } + platform_mutex_unlock(&spl->checkpoint_state_lock); + + if (pending != NULL) { + log_deinit(pending); + shard_log_dec_ref(spl->cc, &pending_head); + } + + core_checkpoint_fill_result(spl, 0, result); + return STATUS_OK; +} + +/* Return one atomic semantic view of the requested checkpoint ticket. */ +static platform_status +core_checkpoint_observe(core_handle *spl, + uint64 ticket, + core_checkpoint_result *result) +{ + core_checkpoint_fill_result(spl, ticket, result); + return STATUS_OK; +} + +/* + * Detach all checkpoint-owned resources after task and API quiescence. Extents + * are reclaimed only when the caller has first removed their durable + * reachability. + */ +static platform_status +core_checkpoint_cleanup_quiesced(core_handle *spl, + bool32 reclaim_extents, + core_checkpoint_result *result) +{ + log_handle *pending = NULL; + log_handle *to_seal = NULL; + log_head pending_head = {0}; + log_head sealed = {0}; + bool32 release_pending = FALSE; + bool32 release_sealed = FALSE; + + platform_mutex_lock(&spl->checkpoint_state_lock); core_checkpoint_state *cp = &spl->checkpoint; switch (cp->phase) { case CORE_CHECKPOINT_IDLE: break; case CORE_CHECKPOINT_PENDING: - // The next live log was pre-created but never installed; discard it. - log_seal(cp->pending_log); - log_dec_ref(spl->cc, &cp->live_head); + platform_assert(cp->pending_log != NULL); + pending = cp->pending_log; + pending_head = log_get_head(pending); + release_pending = reclaim_extents; break; + case CORE_CHECKPOINT_SEALING: + to_seal = cp->log_to_seal; + // fallthrough case CORE_CHECKPOINT_INCORPORATING: + sealed = cp->sealed_head; + release_sealed = reclaim_extents; + break; + case CORE_CHECKPOINT_PUBLISHING: case CORE_CHECKPOINT_COMPLETING: - // The sealed log is fully incorporated after quiesce. The shutdown - // publish records sealed=none, so just reclaim its extents here - // (before the map is persisted, so the map reflects the free). - log_dec_ref(spl->cc, &cp->sealed_head); + /* + * These phases mean another advance call owns an out-of-lock effect + * and may still be using the detached resources. Quiescence + * requires that invocation to have settled first. + */ + platform_assert( + FALSE, "active checkpoint phase %d at cleanup", cp->phase); break; - case CORE_CHECKPOINT_SEALING: default: platform_assert( - FALSE, "unexpected checkpoint phase %d at shutdown", cp->phase); + FALSE, "unexpected checkpoint phase %d at cleanup", cp->phase); + } + ZERO_CONTENTS(cp); + platform_mutex_unlock(&spl->checkpoint_state_lock); + + if (pending != NULL) { + log_deinit(pending); + } + if (to_seal != NULL) { + log_deinit(to_seal); + } + if (release_pending) { + shard_log_dec_ref(spl->cc, &pending_head); + } + if (release_sealed) { + shard_log_dec_ref(spl->cc, &sealed); } - ZERO_CONTENTS(cp); // phase == CORE_CHECKPOINT_IDLE + + core_checkpoint_fill_result(spl, 0, result); + return STATUS_OK; +} + +/* + * Act on the size policy from the insert path. Called by core_insert() once + * the insert lock is released. + * + * Crossing the soft threshold arms a checkpoint, then leaves it PENDING for a + * byte grace period so a normal fullness-driven memtable rotation can consume + * it. Overwrite-in-place traffic may never fill a memtable, so once the live + * log consumes that grace this path forces the rotation as a backstop. + * + * CORE_CHECKPOINT_REQUEST_IF_DUE makes this safe against a stale flag. The + * flag is only a hint -- sampled on some earlier insert, and readable by + * several threads at once -- so a thread can arrive here long after the log it + * observed was already cut. core_checkpoint_request() re-checks the policy + * under the state lock against the *current* log and arms only when another cut + * is genuinely due. The conditional force revalidates PENDING and its hard + * byte limit after obtaining insert exclusion, so a natural rotation or a + * competing force cannot make a stale observer rotate the new memtable again. + */ +typedef struct core_automatic_checkpoint_force_context { + core_handle *spl; + uint64 ticket; + uint64 pending_epoch; +} core_automatic_checkpoint_force_context; + +static bool32 +core_automatic_checkpoint_rotation_due(void *arg) +{ + core_automatic_checkpoint_force_context *ctxt = arg; + core_handle *spl = ctxt->spl; + + platform_mutex_lock(&spl->checkpoint_state_lock); + bool32 due = spl->checkpoint.phase == CORE_CHECKPOINT_PENDING + && spl->checkpoint.completions + 1 == ctxt->ticket + && spl->checkpoint.pending_epoch == ctxt->pending_epoch + && spl->checkpoint.force_at_log_size != UINT64_MAX + && log_get_size(spl->log) >= spl->checkpoint.force_at_log_size; + platform_mutex_unlock(&spl->checkpoint_state_lock); + return due; +} + +/* + * Fast path for the potentially long grace window. Most inserts after the + * soft threshold merely observe the same PENDING checkpoint below its hard + * limit; sample that state once rather than running the general advance and + * request machinery on every insert. + */ +static bool32 +core_automatic_checkpoint_observe_pending(core_handle *spl, + core_checkpoint_result *result) +{ + ZERO_CONTENTS(result); + + platform_mutex_lock(&spl->checkpoint_state_lock); + if (spl->checkpoint.phase != CORE_CHECKPOINT_PENDING) { + platform_mutex_unlock(&spl->checkpoint_state_lock); + return FALSE; + } + + result->ticket = spl->checkpoint.completions + 1; + result->pending_epoch = spl->checkpoint.pending_epoch; + result->rotation_pending = TRUE; + result->automatic_rotation_force_due = + spl->checkpoint.force_at_log_size != UINT64_MAX + && log_get_size(spl->log) >= spl->checkpoint.force_at_log_size; + platform_mutex_unlock(&spl->checkpoint_state_lock); + return TRUE; +} + +static void +core_maybe_cut_oversized_log(core_handle *spl) +{ + if (!__atomic_load_n(&spl->log_reached_threshold, __ATOMIC_RELAXED)) { + return; + } + + core_checkpoint_result request; + if (!core_automatic_checkpoint_observe_pending(spl, &request)) { + /* + * A previous size-triggered rotation may have cut the log and then + * failed while sealing or publishing it. Overwrite-in-place traffic + * need not produce another natural rotation, so use the next threshold + * observation to resume that checkpoint before trying to arm a new one. + */ + platform_status rc = core_checkpoint_advance(spl, NULL); + if (!SUCCESS(rc)) { + return; + } + + rc = core_checkpoint_request( + spl, CORE_CHECKPOINT_REQUEST_IF_DUE, 0, &request); + if (!SUCCESS(rc)) { + platform_error_log("core_maybe_cut_oversized_log: could not arm a " + "checkpoint: %s\n", + platform_status_to_string(rc)); + return; + } + } + if (request.automatic_rotation_force_due) { + core_automatic_checkpoint_force_context force_ctxt = { + .spl = spl, + .ticket = request.ticket, + .pending_epoch = request.pending_epoch, + }; + platform_status rc = + memtable_force_rotation_if(&spl->mt_ctxt, + core_automatic_checkpoint_rotation_due, + &force_ctxt, + NULL); + if (!SUCCESS(rc)) { + /* + * A full memtable ring is transient: retain both PENDING and its + * original byte deadline so a later insert can retry. A recorded + * incorporation failure is terminal for this attempt, so release the + * speculative next log rather than leaving it permanently PENDING. + */ + if (!STATUS_IS_EQ(rc, STATUS_BUSY)) { + (void)core_checkpoint_cancel_pending( + spl, request.ticket, request.pending_epoch, NULL); + platform_error_log( + "core_maybe_cut_oversized_log: failed to force a memtable " + "rotation: %s\n", + platform_status_to_string(rc)); + } + } + } +} + +/* + * Report a rotation while its critical section holds the insert lock + * exclusively. core_checkpoint_rotated_locked() swaps the pre-created live + * log in. Every log writer holds the insert lock shared from its group + * reservation through its reserved write, so once this store retires no + * writer is using, or can newly enter, the old log -- making the subsequent + * seal safe. + */ +static void +core_rotate_log(void *arg, uint64 finalized_generation) +{ + core_handle *spl = arg; + + platform_status rc = + core_checkpoint_rotated_locked(spl, finalized_generation, NULL); + platform_assert_status_ok(rc); } /* @@ -854,32 +1287,59 @@ core_begin_memtable_insert(core_handle *spl, uint64 *generation, memtable **mt) return STATUS_OK; } +typedef struct core_log_write_context { + log_handle *log; + log_write_token token; + bool32 reserved; +} core_log_write_context; + +/* + * This is the update's logical linearization point. The btree invokes it + * exactly once with the final leaf write-locked, immediately before the + * guaranteed incorporation. Reserving is allocation- and I/O-free. + */ +static void +core_log_write_reserve(void *arg) +{ + core_log_write_context *ctxt = arg; + platform_assert(ctxt->log != NULL); + platform_assert(!ctxt->reserved); + log_write_reserve(ctxt->log, &ctxt->token); + ctxt->reserved = TRUE; +} + static platform_status core_log_insert(core_handle *spl, uint64 memtable_generation, key tuple_key, message msg, - const btree_insert_results *insert_results) + const btree_insert_results *insert_results, + core_log_write_context *write_ctxt) { - /* TODO: FIXME: One way we could get stuck in a fetch-and-update is if the - * insert succeeds but the lookup fails (e.g. due to an I/O error while - * traversing the trunk). I think the promise we should make in that case is - * that we will preserve enough information in the log to enable the user - * to recover the old value. One way to do this might be to insert a - * reference to the trunk into the log. */ - if (!spl->cfg.use_log) { + /* + * spl->log is NULL while crash recovery replays: the replayed records are + * already in a log, and the session's live log is not cut until replay has + * been folded into a published root. Writing them back out would be pure + * waste, and there would be nowhere to put them. + */ + if (!spl->cfg.use_log || spl->log == NULL) { + platform_assert(!write_ctxt->reserved); return STATUS_OK; } + platform_assert(write_ctxt->reserved); + platform_assert(write_ctxt->log == spl->log); + message log_msg = merge_accumulator_is_null(&insert_results->msg_blob) ? msg : merge_accumulator_to_message(&insert_results->msg_blob); - int log_rc = log_write(spl->log, - tuple_key, - log_msg, - memtable_generation, - insert_results->leaf_generation); + int log_rc = log_write_reserved(&write_ctxt->token, + tuple_key, + log_msg, + memtable_generation, + insert_results->leaf_generation); + write_ctxt->reserved = FALSE; /* * Sample the size policy while we still hold the shared insert lock, which @@ -890,14 +1350,15 @@ core_log_insert(core_handle *spl, * Only ever set it here, never clear it: this runs on every logged insert on * every thread, and writing a shared field that often would bounce its cache * line between cores for no reason. Leaving the common case read-only keeps - * the line shared. core_rotate_log() clears the flag when it cuts the log, - * which it does under the insert lock held exclusively -- so the clear - * cannot race this store. + * the line shared. core_checkpoint_rotated_locked() clears the flag when it + * cuts the log, + * while the insert lock is held exclusively, so the clear cannot race this + * store. */ if (spl->cfg.checkpoint_log_size_bytes != 0 && log_get_size(spl->log) >= spl->cfg.checkpoint_log_size_bytes) { - spl->log_reached_threshold = TRUE; + __atomic_store_n(&spl->log_reached_threshold, TRUE, __ATOMIC_RELAXED); } return log_rc == 0 ? STATUS_OK : (platform_status){.r = log_rc}; @@ -1166,8 +1627,8 @@ core_memtable_flush_internal(core_handle *spl, uint64 generation) generation++; } while (core_try_continue_incorporate(spl, generation)); - // A checkpoint's sealed log may now be fully incorporated; complete it. - core_maybe_complete_checkpoint(spl); + // An incorporation can make a checkpoint eligible for completion. + (void)core_checkpoint_advance(spl, NULL); out: return STATUS_OK; } @@ -1206,15 +1667,11 @@ core_memtable_flush_virtual(void *arg, uint64 generation) { core_handle *spl = arg; - // Begin, step 3: if this rotation's in-CS hook swapped the live log, seal - // the old one now that the critical section has been released. - core_checkpoint_seal_cut(spl); + // Advance any transition this rotation made eligible now that the critical + // section has been released. + (void)core_checkpoint_advance(spl, NULL); core_memtable_flush(spl, generation); - - // Begin, step 1: decide whether the next rotation should start a checkpoint, - // pre-creating its live log outside any critical section. - core_checkpoint_maybe_begin(spl); } static inline uint64 @@ -2025,8 +2482,10 @@ core_insert(core_handle *spl, message data, lookup_result *old_result) { - timestamp ts; - const threadid tid = platform_get_tid(); + timestamp ts; + const threadid tid = platform_get_tid(); + platform_status rc; + if (spl->cfg.use_stats) { ts = platform_get_timestamp(); } @@ -2039,15 +2498,22 @@ core_insert(core_handle *spl, lookup_result_reset(old_result); } - uint64 generation; - memtable *mt = NULL; - platform_status rc = core_begin_memtable_insert(spl, &generation, &mt); + uint64 generation; + memtable *mt = NULL; + rc = core_begin_memtable_insert(spl, &generation, &mt); if (!SUCCESS(rc)) { goto out; } btree_insert_results insert_results; btree_insert_results_init(&insert_results, old_result); + core_log_write_context write_ctxt = { + .log = spl->cfg.use_log ? spl->log : NULL, + }; + if (write_ctxt.log != NULL) { + btree_insert_results_set_callback( + &insert_results, core_log_write_reserve, &write_ctxt); + } rc = memtable_insert(&spl->mt_ctxt, mt, PROCESS_PRIVATE_HEAP_ID, @@ -2055,14 +2521,23 @@ core_insert(core_handle *spl, data, &insert_results); if (!SUCCESS(rc)) { + platform_assert(!write_ctxt.reserved, + "btree insert failed after reserving a log group"); goto end_insert; } - rc = core_log_insert(spl, generation, tuple_key, data, &insert_results); + rc = core_log_insert( + spl, generation, tuple_key, data, &insert_results, &write_ctxt); if (!SUCCESS(rc)) { goto end_insert; } + /* TODO: FIXME: One way we could get stuck in a fetch-and-update is if the + * insert succeeds but the lookup fails (e.g. due to an I/O error while + * traversing the trunk). I think the promise we should make in that case is + * that we will preserve enough information in the log to enable the user + * to recover the old value. One way to do this might be to insert a + * reference to the trunk into the log. */ if (old_result != NULL) { if (lookup_result_should_continue(old_result)) { memtable_begin_lookup(&spl->mt_ctxt); @@ -2353,17 +2828,309 @@ core_create_stats(core_handle *spl) return STATUS_OK; } - spl->stats = core_stats_create(spl->heap_id); - return spl->stats == NULL ? STATUS_NO_MEMORY : STATUS_OK; -} + spl->stats = core_stats_create(spl->heap_id); + return spl->stats == NULL ? STATUS_NO_MEMORY : STATUS_OK; +} + +static void +core_destroy_stats(core_handle *spl) +{ + core_stats_destroy(spl->heap_id, spl->stats); + spl->stats = NULL; +} + + +/* + *----------------------------------------------------------------------------- + * Crash recovery. + * + * A mount whose persisted allocation state is invalid cannot trust the refcount + * map, so it reconstructs one from what is on disk, replays whatever the logs + * hold that the durable root does not, and then publishes a root of its own. + * + * Two passes over the allocator, deliberately. The first counts the logs, so + * that replay -- which allocates -- is never handed an extent that a record it + * has not reached yet depends on. The second, once replay has been folded into + * a published root naming no logs, counts the root alone; the logs are freed by + * being absent from it. See allocator_recovery_begin() for why that beats + * enumerating them a second time to release them. + *----------------------------------------------------------------------------- + */ + +/* Defined below. Recovery uses it to fold replayed records into the tree. */ +static bool32 +core_quiesce(core_handle *spl); + +/* + * Bring the memtable and trunk contexts up over a durable root. Shared by a + * normal mount and by recovery, which drops them and brings them back up over + * the root it publishes. On failure nothing is left initialized. + */ +static platform_status +core_open_contexts(core_handle *spl, uint64 root_addr, uint64 resume_generation) +{ + platform_status rc = + memtable_context_init_at_generation(&spl->mt_ctxt, + spl->heap_id, + spl->cc, + &spl->cfg.mt_cfg, + core_rotate_log, + core_memtable_flush_virtual, + spl, + resume_generation); + if (!SUCCESS(rc)) { + platform_error_log("core_open_contexts: " + "memtable_context_init_at_generation failed: %s\n", + platform_status_to_string(rc)); + return rc; + } + + trunk_snapshot root_snapshot; + rc = trunk_snapshot_create_from_addr(spl->al, root_addr, &root_snapshot); + if (!SUCCESS(rc)) { + platform_error_log("core_open_contexts: " + "trunk_snapshot_create_from_addr failed: %s\n", + platform_status_to_string(rc)); + memtable_context_deinit(&spl->mt_ctxt); + return rc; + } + + // Consumes the snapshot's reference whether or not it succeeds. + rc = trunk_context_init(&spl->trunk_context, + spl->cfg.trunk_node_cfg, + spl->heap_id, + spl->cc, + spl->al, + spl->ts, + root_snapshot); + if (!SUCCESS(rc)) { + platform_error_log("core_open_contexts: trunk_context_init failed: %s\n", + platform_status_to_string(rc)); + memtable_context_deinit(&spl->mt_ctxt); + return rc; + } + return STATUS_OK; +} + +static void +core_close_contexts(core_handle *spl) +{ + platform_status trunk_rc = trunk_context_deinit(&spl->trunk_context); + if (!SUCCESS(trunk_rc)) { + platform_error_log("core_close_contexts: trunk reference cleanup was " + "incomplete; the allocator map will be rebuilt: %s\n", + platform_status_to_string(trunk_rc)); + spl->allocator_map_needs_rebuild = TRUE; + } + memtable_context_deinit(&spl->mt_ctxt); +} + +/* + * Rebuild the refcount map from the durable record. With include_logs, the + * streams the record names are counted too, along with the blobs their + * replayable records point at. + */ +static platform_status +core_rebuild_allocations(core_handle *spl, + const superblock_tree_record *rec, + bool32 include_logs) +{ + /* A partial walk is never eligible to become durable allocator state. */ + spl->allocator_map_needs_rebuild = TRUE; + platform_status rc = allocator_recovery_begin(spl->al); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: allocator_recovery_begin failed: %s\n", + platform_status_to_string(rc)); + return rc; + } + + rc = trunk_recover_allocations( + spl->cfg.trunk_node_cfg, spl->cc, spl->heap_id, rec->root_addr); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: could not rebuild the tree's " + "allocations: %s\n", + platform_status_to_string(rc)); + return rc; + } + + if (include_logs) { + shard_log_config *log_cfg = (shard_log_config *)spl->cfg.log_cfg; + superblock_log_head slots[2] = {rec->sealed_log, rec->live_log}; + for (uint64 i = 0; i < ARRAY_SIZE(slots); i++) { + if (SUPERBLOCK_NO_LOG(slots[i])) { + continue; + } + log_head head = slots[i].head; + + rc = shard_log_recover_allocations(spl->cc, log_cfg, head); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: could not rebuild the allocations " + "of the log at %lu: %s\n", + head.addr, + platform_status_to_string(rc)); + return rc; + } + } + } + + allocator_recovery_finish(spl->al); + spl->allocator_map_needs_rebuild = FALSE; + return STATUS_OK; +} + +/* + * Apply one stream's records to the memtables, skipping those the durable root + * already contains. + * + * Reports through ran_to_end whether the stream reached its end-of-stream + * marker. A caller must not replay a later stream once one has come up short: + * the records of a truncated stream are a valid prefix on their own, but + * anything written after it would be applied on top of a gap. + */ +static platform_status +core_replay_log(core_handle *spl, + log_head head, + uint64 first_unincorporated_generation, + bool32 *ran_to_end) +{ + // An absent slot has no tail to have lost, so it does not stop the next one. + *ran_to_end = TRUE; + if (head.addr == 0) { + return STATUS_OK; + } + + log_iterator *itor; + platform_status rc = + shard_log_iterator_create(spl->cc, + (shard_log_config *)spl->cfg.log_cfg, + spl->heap_id, + head, + first_unincorporated_generation, + &itor); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: could not read the log at %lu for " + "replay: %s\n", + head.addr, + platform_status_to_string(rc)); + return rc; + } + + rc = STATUS_OK; + uint64 applied = 0; + while (SUCCESS(rc) && log_iterator_can_next(itor)) { + key tuple_key; + message msg; + uint64 memtable_generation; + uint64 leaf_generation; + log_iterator_curr(itor, &tuple_key, &msg); + log_iterator_curr_generations( + itor, &memtable_generation, &leaf_generation); + + /* + * The iterator yields records in (memtable generation, leaf generation) + * order, which is the order they were applied in, and has already omitted + * anything below the bound because it is folded into the durable root. + */ + platform_assert(memtable_generation >= first_unincorporated_generation); + rc = core_insert(spl, tuple_key, msg, NULL); + if (SUCCESS(rc)) { + applied++; + } + if (SUCCESS(rc)) { + rc = log_iterator_next(itor); + } + } + + if (SUCCESS(rc)) { + *ran_to_end = log_iterator_stream_complete(itor); + platform_default_log("core_mount: replayed %lu records from the log at " + "%lu%s\n", + applied, + head.addr, + *ran_to_end ? "" : "; its tail was lost"); + } + log_iterator_deinit(itor); + return rc; +} + +/* + * Replay both streams onto the mounted contexts and fold the result into a + * published root naming no logs. The caller then closes these contexts and + * rebuilds the map from that root, which is what releases the logs. + * + * Requires the trunk and memtable contexts to be up, and requires that the + * session's live log has NOT been cut yet: replay must not write the records it + * is reading back out, and the new log must be allocated from the second map. + */ +static platform_status +core_recover_replay(core_handle *spl, const superblock_tree_record *rec) +{ + platform_assert(spl->log == NULL); + + if (!SUPERBLOCK_NO_LOG(rec->live_log) && !spl->cfg.use_log) { + platform_error_log("core_mount: the durable record names a log to replay " + "but logging is disabled, so its records cannot be " + "recovered\n"); + return STATUS_INVALID_STATE; + } + + /* + * Sealed before live: a checkpoint moves the retiring stream into the sealed + * slot, so it holds the older generations. A truncated sealed stream stops + * replay there rather than applying the live stream over the gap. + */ + superblock_log_head slots[2] = {rec->sealed_log, rec->live_log}; + for (uint64 i = 0; i < ARRAY_SIZE(slots); i++) { + bool32 ran_to_end; + platform_status rc = core_replay_log( + spl, slots[i].head, rec->first_unincorporated_generation, &ran_to_end); + if (!SUCCESS(rc)) { + return rc; + } + if (!ran_to_end) { + if (i + 1 < ARRAY_SIZE(slots) && !SUPERBLOCK_NO_LOG(slots[i + 1])) { + platform_error_log("core_mount: the log at %lu lost its tail, so " + "the log after it is not replayable and its " + "records are lost\n", + slots[i].head.addr); + } + break; + } + } + + /* + * Fold everything replayed into the tree. This has to succeed before the + * publish below: a memtable that never incorporated keeps its records in a + * btree that hangs off the memtable context rather than the root, and the + * rebuild that follows -- which counts only the root -- would free it. + */ + if (!core_quiesce(spl)) { + platform_error_log("core_mount: replayed records did not all " + "incorporate; abandoning recovery rather than " + "publishing a root that omits them\n"); + return STATUS_INVALID_STATE; + } + platform_status cleanup_rc = + core_checkpoint_cleanup_quiesced(spl, FALSE, NULL); + platform_assert_status_ok(cleanup_rc); -static void -core_destroy_stats(core_handle *spl) -{ - core_stats_destroy(spl->heap_id, spl->stats); - spl->stats = NULL; -} + /* + * Publish the recovered root with both log slots cleared. From here the + * durable state is exactly what an unmount passes through just before it + * persists the map, so a crash now leaves the next mount rebuilding from a + * root with no logs -- precisely the step below. + */ + superblock_discard_logs(&spl->superblock); + platform_status rc = core_checkpoint_commit_current_root(spl); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: failed to publish the recovered root: " + "%s\n", + platform_status_to_string(rc)); + return rc; + } + return STATUS_OK; +} /* Format the disk and mount the database */ platform_status @@ -2425,11 +3192,11 @@ core_mkfs(core_handle *spl, // set up the log if (spl->cfg.use_log) { - spl->log = shard_log_create( - cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); - if (spl->log == NULL) { - platform_error_log("core_mkfs: shard_log_create failed\n"); - rc = STATUS_NO_MEMORY; + rc = shard_log_create( + cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id, &spl->log); + if (!SUCCESS(rc)) { + platform_error_log("core_mkfs: shard_log_create failed: %s\n", + platform_status_to_string(rc)); goto deinit_memtable_context; } } @@ -2477,10 +3244,15 @@ core_mkfs(core_handle *spl, deinit_stats: core_destroy_stats(spl); deinit_trunk_context: - trunk_context_deinit(&spl->trunk_context); + (void)trunk_context_deinit(&spl->trunk_context); deinit_log: if (spl->cfg.use_log) { - platform_free(spl->heap_id, spl->log); + /* + * A failed superblock publish may nevertheless have reached disk, so do + * not reclaim the stream extents here. Do release the handle's staging + * buffers, writeback set, mutex, and unused mini-allocator reserves. + */ + log_deinit(spl->log); spl->log = NULL; } deinit_memtable_context: @@ -2515,7 +3287,8 @@ core_mount(core_handle *spl, spl->heap_id = hid; spl->ts = ts; - platform_status rc = core_locks_init(spl); + bool32 contexts_open = FALSE; + platform_status rc = core_locks_init(spl); if (!SUCCESS(rc)) { platform_error_log("core_mount: lock initialization failed: %s\n", platform_status_to_string(rc)); @@ -2541,30 +3314,29 @@ core_mount(core_handle *spl, superblock_get_tree_record(&spl->superblock, &rec); /* - * Preserve the historical clean-only mount rule for this first format - * slice: crash recovery (log replay + allocator rebuild) is not wired yet, - * so only a clean at-rest instance -- one whose allocation state is still - * valid -- may supply the root. A valid allocation state is published only - * at the end of a clean unmount, so it is the single at-rest signal (no - * separate per-tree clean flag is needed; see superblock.h). + * A valid persisted allocation state is published only at the end of a clean + * unmount, so it is the single at-rest signal and its absence means a crash + * (no separate per-tree clean flag is needed; see superblock.h). Either the + * map on disk can be trusted, or it has to be rebuilt from what the record + * points at. Both must happen before trunk_snapshot_create_from_addr(), + * which increments the root's refcount in the resulting map. */ - bool32 rebuild = !superblock_allocation_state_valid(&spl->superblock); - if (rebuild) { - platform_error_log("core_mount: root id %lu requires crash recovery\n", - spl->id); - rc = STATUS_INVALID_STATE; - goto deinit_superblock; + bool32 recovering = !superblock_allocation_state_valid(&spl->superblock); + if (recovering) { + platform_default_log("core_mount: root id %lu was not cleanly unmounted; " + "recovering\n", + spl->id); + rc = core_rebuild_allocations(spl, &rec, TRUE); + } else { + rc = allocator_load_refcounts(al); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: allocator_load_refcounts failed: %s\n", + platform_status_to_string(rc)); + } else { + spl->allocator_map_needs_rebuild = FALSE; + } } - - /* - * Load the trusted refcount map (rebuild == FALSE on this path). This must - * precede trunk_snapshot_create_from_addr(), which increments the root's - * refcount in the now-loaded map. - */ - rc = allocator_load_refcounts(al); if (!SUCCESS(rc)) { - platform_error_log("core_mount: allocator_load_refcounts failed: %s\n", - platform_status_to_string(rc)); goto deinit_superblock; } @@ -2573,59 +3345,89 @@ core_mount(core_handle *spl, // exactly where the memtable resumes (0 for a fresh, never-incorporated db). uint64 resume_generation = rec.first_unincorporated_generation; - memtable_config *mt_cfg = &spl->cfg.mt_cfg; - rc = memtable_context_init_at_generation(&spl->mt_ctxt, - spl->heap_id, - cc, - mt_cfg, - core_rotate_log, - core_memtable_flush_virtual, - spl, - resume_generation); + rc = core_open_contexts(spl, root_addr, resume_generation); if (!SUCCESS(rc)) { - platform_error_log("core_mount: memtable_context_init_at_generation " - "failed: %s\n", - platform_status_to_string(rc)); goto deinit_superblock; } + contexts_open = TRUE; - if (spl->cfg.use_log) { - spl->log = shard_log_create( - cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); - if (spl->log == NULL) { - platform_error_log("core_mount: shard_log_create failed\n"); - rc = STATUS_NO_MEMORY; - goto deinit_memtable_context; - } - } - - trunk_snapshot root_snapshot; - rc = trunk_snapshot_create_from_addr(al, root_addr, &root_snapshot); + rc = core_create_stats(spl); if (!SUCCESS(rc)) { - platform_error_log( - "core_mount: trunk_snapshot_create_from_addr failed: %s\n", - platform_status_to_string(rc)); - goto deinit_log; + platform_error_log("core_mount: core_create_stats failed: %s\n", + platform_status_to_string(rc)); + goto deinit_contexts; } - rc = trunk_context_init(&spl->trunk_context, - spl->cfg.trunk_node_cfg, - hid, - cc, - al, - ts, - root_snapshot); - if (!SUCCESS(rc)) { - platform_error_log("core_mount: trunk_context_init failed: %s\n", - platform_status_to_string(rc)); - goto deinit_log; + /* + * Replay, before this session's log exists. Two reasons it has to come + * first: the records being read must not be written straight back out, and + * the new log's extents must come from the map the recovery publish leaves + * behind rather than the one that still counts the logs being replayed. + */ + if (recovering) { + rc = core_recover_replay(spl, &rec); + if (!SUCCESS(rc)) { + /* + * Replay may have rotated memtables and queued their flush or + * incorporation before encountering the bad record. Drain that work + * while its stats, memtable, and trunk contexts are still alive; + * common mount cleanup may then tear those contexts down safely. + */ + (void)core_quiesce(spl); + goto deinit_stats; + } + + /* + * The contexts hold references that a root-only rebuild must not count. + * Close them in this scope so every failure path below knows whether the + * common cleanup still owns live contexts; core_recover_replay() itself + * leaves them open on every return. + */ + core_close_contexts(spl); + contexts_open = FALSE; + + /* + * The root-only allocator rebuild below makes the replayed log and blob + * extents free. Remove their old cache mappings first, while the + * root-plus-logs recovery map still owns every resident address; + * otherwise immediate address reuse could create two cache entries for + * one page. core_recover_replay() has already written back and durably + * published the recovered root, and closing the contexts made the cache + * quiescent. + */ + rc = cache_evict(spl->cc, FALSE /* ignore_pinned_pages */); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: failed to invalidate the cache before " + "the root-only allocation rebuild: %s\n", + platform_status_to_string(rc)); + goto deinit_stats; + } + + // The recovery publish advanced the root and cleared both log slots. + superblock_get_tree_record(&spl->superblock, &rec); + rc = core_rebuild_allocations(spl, &rec, FALSE); + if (SUCCESS(rc)) { + rc = core_open_contexts( + spl, rec.root_addr, rec.first_unincorporated_generation); + if (SUCCESS(rc)) { + contexts_open = TRUE; + } + } + + if (!SUCCESS(rc)) { + goto deinit_stats; + } + resume_generation = rec.first_unincorporated_generation; } - rc = core_create_stats(spl); - if (!SUCCESS(rc)) { - platform_error_log("core_mount: core_create_stats failed: %s\n", - platform_status_to_string(rc)); - goto deinit_trunk_context; + if (spl->cfg.use_log) { + rc = shard_log_create( + cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id, &spl->log); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: shard_log_create failed: %s\n", + platform_status_to_string(rc)); + goto deinit_stats; + } } /* @@ -2646,21 +3448,23 @@ core_mount(core_handle *spl, platform_error_log("core_mount: mark-dirty superblock_make_durable " "failed: %s\n", platform_status_to_string(rc)); - goto deinit_stats; + goto deinit_log; } return STATUS_OK; -deinit_stats: - core_destroy_stats(spl); -deinit_trunk_context: - trunk_context_deinit(&spl->trunk_context); + // The log is created last of all here, so it unwinds first. deinit_log: - if (spl->cfg.use_log) { - platform_free(spl->heap_id, spl->log); + if (spl->cfg.use_log && spl->log != NULL) { + /* See the corresponding mkfs unwind: publication may be ambiguous. */ + log_deinit(spl->log); spl->log = NULL; } -deinit_memtable_context: - memtable_context_deinit(&spl->mt_ctxt); +deinit_stats: + core_destroy_stats(spl); +deinit_contexts: + if (contexts_open) { + core_close_contexts(spl); + } deinit_superblock: superblock_context_deinit(&spl->superblock); deinit_locks: @@ -2668,6 +3472,38 @@ core_mount(core_handle *spl, return rc; } +/* + * Does any memtable still hold records the durable root will not contain? + * + * Deliberately side-effect free, so that core_unmount() can consult it while + * deciding whether to go through with the unmount at all. + * core_report_unincorporated_memtables() walks the same generations but also + * logs and releases, so it is only safe once teardown is committed to. + */ +static bool32 +core_have_unincorporated_memtables(core_handle *spl) +{ + uint64 start_generation = memtable_generation_retired(&spl->mt_ctxt) + 1; + uint64 end_generation = memtable_generation(&spl->mt_ctxt); + + for (uint64 generation = start_generation; generation < end_generation; + generation++) + { + memtable *mt = core_try_get_memtable(spl, generation); + if (mt != NULL && mt->state != MEMTABLE_STATE_READY) { + return TRUE; + } + } + return FALSE; +} + +/* + * Report every unincorporated memtable and release the compacted branch each + * one left behind. That release is why this is not a query: calling it and + * then continuing to run would leave the memtable pointing at a branch whose + * reference is gone. Use core_have_unincorporated_memtables() to look without + * touching anything. + */ static bool32 core_report_unincorporated_memtables(core_handle *spl) { @@ -2719,46 +3555,52 @@ core_report_unincorporated_memtables(core_handle *spl) * It intentionally leaves the memtable and log contexts live: the clean * checkpoint record needs both after final incorporation has quiesced. */ -static void -core_quiesce_for_shutdown(core_handle *spl) +/* + * Returns FALSE if any memtable is still unincorporated, i.e. holds records the + * durable root will not contain. + * + * Everything this does is recoverable-from: it finishes outstanding work but + * dismantles nothing, so a caller that does not like the answer may still + * decline to unmount and keep running. + */ +static bool32 +core_quiesce(core_handle *spl) { - // write current memtable to disk - // (any others must already be flushing/flushed) + /* + * Drain older generations before forcing the active one. In addition to + * being work quiesce needs to do anyway, this normally recycles the next + * ring slot and makes the forced rotation immediately possible. + */ + platform_status rc = task_perform_until_quiescent(spl->ts); + platform_assert_status_ok(rc); if (!memtable_is_empty(&spl->mt_ctxt)) { /* - * memtable_force_rotation is not thread safe. It dispatches the flush - * itself (via the process callback), which also resolves any checkpoint - * log cut its rotate hook just made. That callback may arm a fresh - * checkpoint; core_checkpoint_cleanup_for_shutdown() discards it. + * The checked force can still report BUSY if an older generation failed + * incorporation and therefore could not recycle its ring slot. Do not + * pretend the active generation was incorporated: it remains outside the + * generation range inspected by core_have_unincorporated_memtables(). */ - memtable_force_rotation(&spl->mt_ctxt); - } - - // finish any outstanding tasks and destroy task system for this table. - platform_status rc = task_perform_until_quiescent(spl->ts); - platform_assert_status_ok(rc); - - core_report_unincorporated_memtables(spl); -} + rc = memtable_force_rotation(&spl->mt_ctxt, NULL); + if (!SUCCESS(rc)) { + if (STATUS_IS_EQ(rc, STATUS_BUSY)) { + platform_error_log("core_quiesce: cannot rotate the active " + "memtable because the next ring slot is still " + "in use\n"); + } else { + platform_error_log("core_quiesce: failed to rotate the active " + "memtable: %s\n", + platform_status_to_string(rc)); + } + return FALSE; + } -/* - * Seal the live log at shutdown -- finalizes its pages and frees the handle -- - * and return its identity so the caller can free the extents with log_dec_ref() - * once the cache is flushed. A clean shutdown has folded everything into the - * durable root, so the live log is fully incorporated and discardable. Returns - * an empty descriptor when logging is disabled. - */ -static log_head -core_seal_live_log(core_handle *spl) -{ - if (!spl->cfg.use_log || spl->log == NULL) { - return (log_head){0}; + // The force dispatched the active generation; finish that work too. + rc = task_perform_until_quiescent(spl->ts); + platform_assert_status_ok(rc); } - log_head info = log_get_head(spl->log); - log_seal(spl->log); - spl->log = NULL; - return info; + + return !core_have_unincorporated_memtables(spl); } /* @@ -2790,19 +3632,69 @@ core_seal_live_log(core_handle *spl) * spending a generation and a log extent each time. A longer timeout lets real * traffic drive the cut instead. */ +typedef struct core_checkpoint_rotation_context { + core_handle *spl; + uint64 target_generation; + uint64 ticket; +} core_checkpoint_rotation_context; + +/* Called with memtable inserts excluded by memtable_force_rotation_if(). */ +static bool32 +core_checkpoint_rotation_still_needed(void *arg) +{ + core_checkpoint_rotation_context *ctxt = arg; + if (memtable_generation(&ctxt->spl->mt_ctxt) == ctxt->target_generation) { + return TRUE; + } + + platform_mutex_lock(&ctxt->spl->checkpoint_state_lock); + bool32 pending = ctxt->ticket != 0 + && ctxt->spl->checkpoint.phase == CORE_CHECKPOINT_PENDING + && ctxt->spl->checkpoint.completions + 1 == ctxt->ticket; + platform_mutex_unlock(&ctxt->spl->checkpoint_state_lock); + return pending; +} + platform_status core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) { uint64 target = memtable_generation(&spl->mt_ctxt); - uint64 ticket = core_checkpoint_begin(spl, TRUE /* force */); + core_checkpoint_result request; + platform_status rc = core_checkpoint_request( + spl, CORE_CHECKPOINT_REQUEST_REQUIRED, 0, &request); + if (!SUCCESS(rc)) { + return rc; + } + uint64 ticket = request.ticket; uint64 wait = 100; timestamp deadline = platform_get_timestamp(); while (TRUE) { bool32 incorporated = memtable_generation_retired(&spl->mt_ctxt) + 1 > target; - core_checkpoint_status status = core_checkpoint_status_get(spl); + core_checkpoint_result state; + rc = core_checkpoint_observe(spl, ticket, &state); + if (!SUCCESS(rc)) { + return rc; + } + + /* + * An automatic size-triggered attempt can cancel a still-PENDING + * checkpoint after a terminal rotation failure. If this call had + * attached to that completion, re-arm the same completion ticket instead + * of waiting forever or returning without a log cut. + */ + if (state.ticket_needs_rearm) { + core_checkpoint_result replacement; + rc = core_checkpoint_request( + spl, CORE_CHECKPOINT_REQUEST_REQUIRED, ticket, &replacement); + if (!SUCCESS(rc)) { + return rc; + } + platform_assert(replacement.ticket == ticket); + state = replacement; + } /* * Done once the target is durable-able and, if we started a checkpoint, * that specific checkpoint has completed -- which is what freed its @@ -2810,8 +3702,7 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) * flight" means neither an unrelated checkpoint nor one armed after ours * can hold us up. */ - bool32 ours_completed = (ticket == 0) || (status.completions >= ticket); - if (incorporated && ours_completed) { + if (incorporated && state.ticket_complete) { break; } @@ -2826,14 +3717,42 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) * cut. On an otherwise idle system nothing would ever provide one, and * the wait for our completion would never finish. */ - bool32 needs_rotation = - memtable_generation(&spl->mt_ctxt) == target - || (ticket != 0 && status.phase == CORE_CHECKPOINT_PENDING); + bool32 needs_rotation = memtable_generation(&spl->mt_ctxt) == target + || (ticket != 0 && state.rotation_pending); if (needs_rotation && rotation_timeout_ns <= platform_timestamp_elapsed(deadline)) { - memtable_force_rotation(&spl->mt_ctxt); // dispatches the flush itself - deadline = platform_get_timestamp(); + core_checkpoint_rotation_context rotation_ctxt = { + .spl = spl, + .target_generation = target, + .ticket = ticket, + }; + + platform_status rotation_rc = + memtable_force_rotation_if(&spl->mt_ctxt, + core_checkpoint_rotation_still_needed, + &rotation_ctxt, + NULL); + if (SUCCESS(rotation_rc)) { + /* A no-op means another rotation already supplied the progress. */ + deadline = platform_get_timestamp(); + } else if (!STATUS_IS_EQ(rotation_rc, STATUS_BUSY)) { + (void)core_checkpoint_cancel_pending( + spl, ticket, state.pending_epoch, NULL); + return rotation_rc; + } + } + + /* + * Drive any eligible cut or completion ourselves. This is a no-op when + * another thread owns the transition. On an otherwise idle system no + * later rotation or incorporation may arrive to retry a failed + * transition, so the synchronous caller must both make progress and + * observe any error. + */ + rc = core_checkpoint_advance(spl, NULL); + if (!SUCCESS(rc)) { + return rc; } task_perform_one_if_needed(spl->ts, 0); @@ -2852,69 +3771,262 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) return core_checkpoint_commit_current_root(spl); } +platform_status +core_durable_barrier(core_handle *spl) +{ + /* Without a WAL, the COW root is the only available durability route. */ + if (!spl->cfg.use_log) { + return core_checkpoint(spl, 0); + } + + log_handle *live = NULL; + log_durable_ticket log_ticket = 0; + uint64 publication_target = 0; + platform_status rc; + + /* + * Pin the live-log pointer against memtable rotation while taking the cut. + * This is a shared insert slot, so writers continue to reserve and append + * concurrently. The reservation callback under each final leaf lock is + * what orders visible updates with the group swap below. + */ + memtable_begin_insert(&spl->mt_ctxt); + if (spl->log == NULL) { + rc = STATUS_INVALID_STATE; + goto end_insert_epoch; + } + live = spl->log; + + platform_mutex_lock(&spl->checkpoint_state_lock); + if (spl->checkpoint.phase == CORE_CHECKPOINT_SEALING + || spl->checkpoint.phase == CORE_CHECKPOINT_PUBLISHING) + { + publication_target = spl->checkpoint.publications + 1; + platform_assert(publication_target != 0); + } + platform_mutex_unlock(&spl->checkpoint_state_lock); + + rc = log_make_durable_begin(live, &log_ticket); + +end_insert_epoch: + memtable_end_insert(&spl->mt_ctxt); + if (!SUCCESS(rc)) { + return rc; + } + + /* wait consumes the ticket and its pin on every return path. */ + rc = log_make_durable_wait(live, log_ticket); + if (!SUCCESS(rc)) { + return rc; + } + + if (publication_target != 0) { + rc = core_checkpoint_wait_for_publication(spl, publication_target); + } + return rc; +} + /* * Close (unmount) a database without destroying it. - * It can be re-opened later with core_mount(). + * It can be re-opened later with core_mount(). See core.h for the contract. */ platform_status -core_unmount(core_handle *spl) +core_unmount(core_handle *spl, bool32 force) { - platform_status rc; + /* + * Everything through the data_safe decision is non-destructive. Quiescing + * leaves the memtable, checkpoint, and log contexts live so a failed + * non-forced close can return a usable handle. + */ + bool32 all_incorporated = core_quiesce(spl); /* - * Quiescing leaves the memtable and log contexts live so publication can - * atomically capture the retired generation and root. Teardown is safe - * regardless of publication success. + * A failed checkpoint cut can leave the newly installed live log unnamed by + * the durable superblock. Retry any eligible checkpoint transition once + * before testing the log route. This is still safe to walk away from: + * sealing retires only the previous stream, while inserts already use the + * new one. */ - core_quiesce_for_shutdown(spl); + core_checkpoint_result checkpoint_view; + platform_status checkpoint_rc = + core_checkpoint_advance(spl, &checkpoint_view); + + platform_status log_rc = STATUS_OK; + bool32 have_log = spl->cfg.use_log && spl->log != NULL; + log_head live_log = {0}; + bool32 log_named = FALSE; + bool32 log_durable = FALSE; + if (have_log) { + live_log = log_get_head(spl->log); + superblock_tree_record durable_rec; + superblock_get_tree_record(&spl->superblock, &durable_rec); + log_named = + core_superblock_log_head_matches(durable_rec.live_log, live_log); - // Reclaim any in-flight checkpoint's logs before the unmount publish. - core_checkpoint_cleanup_for_shutdown(spl); + /* + * A cut whose old stream was sealed but whose publication failed leaves + * that stream durably named as live. It is a complete recovery route + * only when the newly installed, unnamed stream has accepted no records. + */ + bool32 retiring_log_durable = + checkpoint_view.unpublished_sealed_log && log_is_empty(spl->log) + && core_superblock_log_head_matches(durable_rec.live_log, + checkpoint_view.retiring_log); + + log_rc = log_make_durable(spl->log); + if (!SUCCESS(log_rc)) { + platform_error_log("core_unmount: failed to make the live log " + "durable: %s\n", + platform_status_to_string(log_rc)); + } + /* + * make_durable() also distinguishes a genuinely empty new stream from + * one whose first reserved append failed before accepting a record. In + * the latter case log_is_empty() is still true, but the poisoned stream + * must not let the still-named retiring log stand in for the missing + * update. + */ + log_durable = SUCCESS(log_rc) && (log_named || retiring_log_durable); + } /* - * The clean-unmount root incorporates everything, so the live log is fully - * folded and discardable. Seal it (frees the handle) now; free its extents - * after the cache flush below. + * First publish the current root with log reachability unchanged. This is + * the safety probe: after an indeterminate write/barrier outcome, both the + * previous record and the candidate still name the same live log. A failed + * non-forced close can therefore restore its in-memory before-image and keep + * running without allowing later writes to disappear into an unnamed log. */ - log_head live_log = core_seal_live_log(spl); + bool32 existing_root_anchor = + all_incorporated && core_current_root_matches_durable_record(spl); + platform_status root_publish_rc = core_checkpoint_commit_current_root(spl); + bool32 root_publish_succeeded = SUCCESS(root_publish_rc); + if (!root_publish_succeeded) { + platform_error_log("core_unmount: failed to publish the unmount root: " + "%s\n", + platform_status_to_string(root_publish_rc)); + } + + bool32 root_anchor = + all_incorporated && (existing_root_anchor || root_publish_succeeded); /* - * Part A: publish the clean-unmount root with both log slots cleared (no - * live or sealed log at rest -- their extents are freed just below, so no - * reference to them may survive). Both transitions go out in the single - * publish the commit performs. Allocation state stays invalid here; it - * becomes valid only in Part B, after the map is persisted. + * Once a complete root is confirmed, clear the log slots in a second + * publication. Failure here cannot endanger data: every possible record + * points at that same complete root. It only determines whether log extents + * may be reclaimed and allocator state may be published as clean. */ - superblock_discard_logs(&spl->superblock); - rc = core_checkpoint_commit_current_root(spl); - if (!SUCCESS(rc)) { - platform_error_log("core_unmount: failed to publish unmount root: %s\n", - platform_status_to_string(rc)); + bool32 logs_discarded = FALSE; + if (all_incorporated && root_publish_succeeded) { + superblock root_superblock; + core_superblock_save_image(spl, &root_superblock); + superblock_discard_logs(&spl->superblock); + platform_status discard_rc = core_checkpoint_commit_current_root(spl); + if (SUCCESS(discard_rc)) { + logs_discarded = TRUE; + } else { + core_superblock_restore_image(spl, &root_superblock); + platform_error_log("core_unmount: failed to publish log removal; " + "the next mount will recover: %s\n", + platform_status_to_string(discard_rc)); + } + } + + /* + * The return value describes data preservation, not whether recovery is + * needed. A confirmed complete root or a synced, durably named live log is + * sufficient. Merely syncing a newly swapped-in but unpublished log is + * not: recovery would have no pointer with which to find it. + */ + bool32 data_safe = root_anchor || log_durable; + platform_status safety_rc = STATUS_OK; + if (!data_safe) { + if (!all_incorporated && have_log && !SUCCESS(log_rc)) { + safety_rc = log_rc; + } else if (!root_publish_succeeded) { + safety_rc = root_publish_rc; + } else if (!SUCCESS(checkpoint_rc)) { + safety_rc = checkpoint_rc; + } else { + safety_rc = STATUS_BUSY; + } + + platform_error_log( + "core_unmount: data preservation could not be guaranteed: %s%s.\n", + !all_incorporated ? "the durable root omits unincorporated memtables" + : "the current root was not durably published", + !log_durable + ? (have_log && !log_named + ? " and the live log is not named by the durable superblock" + : " and no durable log recovery route is available") + : ""); + if (!force) { + platform_error_log("core_unmount: the database remains mounted; " + "retry the unmount or force it\n"); + return safety_rc; + } + } + + /* + * Past this point the unmount is committed to and teardown is destructive. + */ + if (!all_incorporated) { + core_report_unincorporated_memtables(spl); + } + + /* + * Reclaim logs only after a confirmed root publication both covers all + * memtables and durably clears the log slots. In every fallback/recovery + * case retain their extents; the invalid allocation-state marker makes the + * next mount reconstruct exactly what the durable record still reaches. + */ + bool32 reclaim_logs = logs_discarded; + platform_status checkpoint_cleanup_rc = + core_checkpoint_cleanup_quiesced(spl, reclaim_logs, NULL); + platform_assert_status_ok(checkpoint_cleanup_rc); + + /* + * Deliberately no seal of the current live stream. make_durable above + * closed its current group; recovery already treats a live stream as + * possibly truncated, so a terminator adds no safety at shutdown. + */ + if (have_log) { + log_deinit(spl->log); + spl->log = NULL; + } + + if (data_safe && !logs_discarded) { + platform_error_log("core_unmount: shutdown retained recovery state; " + "the next mount must recover\n"); } - // Keep this after publication above: it supplies the generation cut. + // Keep this after root publication: the context supplies its generation cut. memtable_context_deinit(&spl->mt_ctxt); - // Flush all dirty pages. The live log has already been sealed (above); free - // its extents now that the cache is flushed, before the map is persisted - // (Part B) so the persisted map reflects the free. + // Free log extents only after the durable record no longer names them. cache_flush(spl->cc); - log_dec_ref(spl->cc, &live_log); + if (reclaim_logs) { + shard_log_dec_ref(spl->cc, &live_log); + } /* * Release the context's live root reference before persisting the map, so * the persisted refcounts reflect exactly the durable record's single * reference to the root. */ - trunk_context_deinit(&spl->trunk_context); + platform_status trunk_rc = trunk_context_deinit(&spl->trunk_context); + if (!SUCCESS(trunk_rc)) { + platform_error_log("core_unmount: trunk reference cleanup was " + "incomplete; the allocator map will be rebuilt: %s\n", + platform_status_to_string(trunk_rc)); + spl->allocator_map_needs_rebuild = TRUE; + } /* - * Part B: only after a clean Part A publish, persist the refcount map and - * republish a valid allocation state pointing at it. The map is made - * durable before the "map is trustworthy" flag, and that flag becomes - * durable only after the root it must agree with (Part A). On a Part A - * failure we leave the allocation state invalid so the next open rebuilds. + * Part B is only an optimization for the next mount. Publish the map after + * a complete-root Part A only when every reference release was accounted + * for. A suspect map, preserved logs, or a Part-B I/O failure merely forces + * recovery; none changes the data-safety result returned by this function. */ - if (SUCCESS(rc)) { + if (logs_discarded && !spl->allocator_map_needs_rebuild) { uint64 map_addr; platform_status prc = allocator_persist(spl->al, &map_addr); if (SUCCESS(prc)) { @@ -2923,16 +4035,19 @@ core_unmount(core_handle *spl) } if (!SUCCESS(prc)) { platform_error_log("core_unmount: failed to publish clean allocation " - "state: %s\n", + "state; the next mount will rebuild it: %s\n", platform_status_to_string(prc)); - rc = prc; } + } else if (logs_discarded && spl->allocator_map_needs_rebuild) { + platform_error_log("core_unmount: allocator reference accounting is " + "incomplete; leaving allocation state invalid for " + "rebuild on the next mount\n"); } superblock_context_deinit(&spl->superblock); core_destroy_stats(spl); core_locks_deinit(spl); - return rc; + return data_safe ? STATUS_OK : safety_rc; } /* @@ -2941,10 +4056,18 @@ core_unmount(core_handle *spl) void core_destroy(core_handle *spl) { - core_quiesce_for_shutdown(spl); + /* + * Nothing here needs to survive, so an unincorporated memtable is moot -- + * but still report it and release the branch it stranded, since the + * reporting walk is also what cleans up after one. + */ + (void)core_quiesce(spl); + (void)core_report_unincorporated_memtables(spl); // Reclaim any in-flight checkpoint's logs before teardown. - core_checkpoint_cleanup_for_shutdown(spl); + platform_status checkpoint_cleanup_rc = + core_checkpoint_cleanup_quiesced(spl, TRUE, NULL); + platform_assert_status_ok(checkpoint_cleanup_rc); /* * Release the reference the published tree record holds on its root before @@ -2966,12 +4089,19 @@ core_destroy(core_handle *spl) } } - // Discard the live log too: seal (frees the handle), then free its extents - // after the cache flush. - log_head live_log = core_seal_live_log(spl); + /* + * Discard the live log too. Not sealed: the database is being destroyed and + * these extents are about to be freed, so a terminator would serve no one. + */ + log_head live_log = {0}; + if (spl->cfg.use_log && spl->log != NULL) { + live_log = log_get_head(spl->log); + log_deinit(spl->log); + spl->log = NULL; + } memtable_context_deinit(&spl->mt_ctxt); cache_flush(spl->cc); - log_dec_ref(spl->cc, &live_log); + shard_log_dec_ref(spl->cc, &live_log); trunk_context_deinit(&spl->trunk_context); /* @@ -3028,19 +4158,21 @@ core_print_super_block(platform_log_handle *log_handle, core_handle *spl) platform_log(log_handle, "Superblock tree record root_id=%lu {\n" " root_addr=%lu first_unincorporated_generation=%lu\n" - " live_log: meta_addr=%lu addr=%lu magic=%lu\n" - " sealed_log: meta_addr=%lu addr=%lu magic=%lu\n" + " live_log: meta_addr=%lu addr=%lu nonce=%016lx%016lx\n" + " sealed_log: meta_addr=%lu addr=%lu nonce=%016lx%016lx\n" " allocation_state: %s (addr=%lu)\n" "}\n\n", spl->id, rec.root_addr, rec.first_unincorporated_generation, - rec.live_log.meta_addr, - rec.live_log.addr, - rec.live_log.magic, - rec.sealed_log.meta_addr, - rec.sealed_log.addr, - rec.sealed_log.magic, + rec.live_log.head.meta_addr, + rec.live_log.head.addr, + rec.live_log.head.nonce.high, + rec.live_log.head.nonce.low, + rec.sealed_log.head.meta_addr, + rec.sealed_log.head.addr, + rec.sealed_log.head.nonce.high, + rec.sealed_log.head.nonce.low, superblock_allocation_state_valid(&spl->superblock) ? "valid" : "invalid", superblock_allocation_state_addr(&spl->superblock)); @@ -3332,6 +4464,7 @@ core_config_init(core_config *core_cfg, uint64 prefetch_budget, bool32 use_log, uint64 checkpoint_log_size_bytes, + uint64 checkpoint_log_grace_bytes, bool32 use_stats, bool32 verbose_logging, platform_log_handle *log_handle) @@ -3346,13 +4479,14 @@ core_config_init(core_config *core_cfg, core_cfg->trunk_node_cfg = trunk_node_cfg; core_cfg->log_cfg = log_cfg; - core_cfg->queue_scale_percent = queue_scale_percent; - core_cfg->prefetch_budget = prefetch_budget; - core_cfg->use_log = use_log; - core_cfg->checkpoint_log_size_bytes = checkpoint_log_size_bytes; - core_cfg->use_stats = use_stats; - core_cfg->verbose_logging_enabled = verbose_logging; - core_cfg->log_handle = log_handle; + core_cfg->queue_scale_percent = queue_scale_percent; + core_cfg->prefetch_budget = prefetch_budget; + core_cfg->use_log = use_log; + core_cfg->checkpoint_log_size_bytes = checkpoint_log_size_bytes; + core_cfg->checkpoint_log_grace_bytes = checkpoint_log_grace_bytes; + core_cfg->use_stats = use_stats; + core_cfg->verbose_logging_enabled = verbose_logging; + core_cfg->log_handle = log_handle; memtable_config_init(&core_cfg->mt_cfg, core_cfg->btree_cfg, diff --git a/src/core.h b/src/core.h index f4264891..7ee10368 100644 --- a/src/core.h +++ b/src/core.h @@ -52,10 +52,12 @@ typedef struct core_config { bool32 use_log; log_config *log_cfg; /* - * Automatic-checkpoint policy: take a checkpoint (rotate the log and advance - * the durable root) once the live log reaches this many bytes. 0 disables - * automatic checkpoints, leaving durability and log reclamation entirely to - * explicit core_checkpoint() calls. Defaults to the cache size. + * Automatic-checkpoint policy: arm a checkpoint once the live log reaches + * this many bytes. The next natural memtable rotation cuts the log; if none + * arrives within checkpoint_log_grace_bytes, continued log growth forces + * one. 0 disables automatic checkpoints, leaving durability and log + * reclamation entirely to explicit core_checkpoint() calls. The public API + * resolves its zero default to the cache size before initializing core. * * Sizing the trigger by log bytes rather than by memtable generations * matters because the two are independent: a workload that repeatedly @@ -64,7 +66,14 @@ typedef struct core_config { * to the log. A generation-based trigger would never fire and the log would * grow without bound. */ - uint64 checkpoint_log_size_bytes; + uint64 checkpoint_log_size_bytes; + /* + * Additional live-log bytes allowed after an automatic checkpoint is armed + * while waiting for a natural memtable rotation. At this internal layer, 0 + * means no grace and UINT64_MAX disables the forced-rotation backstop; the + * public API resolves its zero default to twice the memtable capacity. + */ + uint64 checkpoint_log_grace_bytes; trunk_config *trunk_node_cfg; // verbose logging @@ -123,25 +132,36 @@ typedef struct core_handle core_handle; * IDLE no checkpoint in progress. * PENDING the next live log is pre-created; the next memtable rotation * will swap it in under the insert lock. - * SEALING the rotation swapped the new live log in; the old log still - * needs sealing (which will be performed just after the - * rotation critical section). - * INCORPORATING the old log is sealed; waiting for its generations to - * be incorporated into the trunk root. + * SEALING the rotation swapped the new live log in; the old log must be + * sealed if needed and the cut still needs publication. + * PUBLISHING a thread has claimed that work and is sealing the old log and + * publishing the cut. Distinct from INCORPORATING because the + * two differ in exactly the way completion cares about: only + * once the cut is published may the sealed log's extents be + * freed, and a concurrent checkpoint advance would otherwise + * be free to complete mid-publish and release extents the + * superblock still names as live. It is also where a failed + * seal returns from: the phase goes back to SEALING, leaving + * the checkpoint exactly as the rotation left it, to be + * retried by a later advance call. + * INCORPORATING the old log is sealed and the cut is published; waiting for + * its generations to be incorporated into the trunk root. * COMPLETING the completion publish (advance root, clear sealed slot) is * in flight. * * The only transition that touches the shared spl->log pointer (PENDING -> * SEALING) runs inside the memtable rotation critical section, where the insert - * lock is held exclusively; every log writer holds that lock shared across its - * log_write, so no writer can be mid-write to, or newly enter, the old log once - * it is swapped out. All other fields are guarded by checkpoint_state_lock, - * which is only ever held for brief, I/O-free updates. + * lock is held exclusively; every log writer holds that lock shared from its + * group reservation through its reserved write, so no writer can still use, + * or newly enter, the old log once it is swapped out. All live transitions + * and observations enter through the event-specific checkpoint functions in + * core.c. The state lock is only ever held for brief, I/O-free updates. */ typedef enum core_checkpoint_phase { CORE_CHECKPOINT_IDLE = 0, CORE_CHECKPOINT_PENDING, CORE_CHECKPOINT_SEALING, + CORE_CHECKPOINT_PUBLISHING, CORE_CHECKPOINT_INCORPORATING, CORE_CHECKPOINT_COMPLETING, } core_checkpoint_phase; @@ -149,22 +169,40 @@ typedef enum core_checkpoint_phase { typedef struct core_checkpoint_state { core_checkpoint_phase phase; log_handle *pending_log; // next live log, pre-created (PENDING) - log_handle *log_to_seal; // old live log awaiting seal (SEALING) - log_head sealed_head; // identity of the sealed log (reclaim) - log_head live_head; // identity of the new live log + /* Monotonic identity of each installed PENDING checkpoint. */ + uint64 pending_epoch; + /* + * Automatic PENDING checkpoints force a rotation once the current live log + * reaches this size. UINT64_MAX means the pending checkpoint was requested + * explicitly, or automatic forced rotation is disabled. The value is + * recorded when PENDING is installed so every concurrent observer uses the + * same grace interval. + */ + uint64 force_at_log_size; + // Old live log awaiting seal (SEALING), or being sealed (PUBLISHING). Kept + // across a failed attempt so the retry has something to resume. + log_handle *log_to_seal; + log_head sealed_head; // identity of the sealed log (reclaim) // First generation the new live log receives, recorded in the superblock as // its coverage start. The retiring log's start needs no tracking: the // superblock already holds it and carries it into the sealed slot. uint64 live_start_generation; uint64 cut_generation; // complete once retired >= this + /* + * Log cuts durably published so far. A durability barrier that observes + * SEALING/PUBLISHING waits for the next value; unlike `completions`, this + * advances as soon as the superblock names both sides of the cut and does + * not wait for incorporation or log reclamation. + */ + uint64 publications; /* * Checkpoints completed so far. Bumped only after the completion has freed * the retired log, so it is the one observable meaning "that checkpoint's * space is back" -- every superblock-visible signal is necessarily written * before the free, since the superblock must stop naming a log before its - * extents are released. core_checkpoint_begin() hands out `completions + 1` - * as a ticket so a caller can wait for its own checkpoint rather than merely - * for "none in flight." + * extents are released. A checkpoint REQUEST hands out `completions + 1` + * as a ticket so a caller can wait for its own checkpoint rather than + * merely for "none in flight." */ uint64 completions; } core_checkpoint_state; @@ -206,6 +244,21 @@ struct core_handle { platform_mutex superblock_lock; superblock_context superblock; + /* + * TRUE while the live allocator map may conservatively overcount extents -- + * for example, after incomplete reference cleanup or after retaining the + * root of an indeterminate superblock publication. The map remains safe for + * ordinary allocation/refcount operations, but publishing it would make the + * leak permanent. A complete recovery walk clears the bit; until then a + * clean shutdown leaves allocation_state invalid so the next mount rebuilds + * the map. A newly initialized map or one loaded from trusted durable state + * starts clear. + * + * This is per-core while an instance owns exactly one tree. It must move + * with allocator/superblock ownership if that changes. + */ + bool32 allocator_map_needs_rebuild; + /* * Incorporation-driven checkpoint state. checkpoint_state_lock guards the * fields of `checkpoint` (and is held while `log` is swapped); it is only @@ -221,8 +274,8 @@ struct core_handle { * core_log_insert() -- which already holds the shared insert lock, the same * lock that excludes the log swap, so it can read `log` safely -- and acted * on by core_insert() once that lock is released. Cleared only by - * core_rotate_log() when it cuts the log, under the insert lock held - * exclusively, so set and clear cannot race. + * core_checkpoint_rotated_locked() when it cuts the log, under the insert + * lock held exclusively, so set and clear cannot race. * * Set-only on the insert path so the common case does no store and the cache * line stays shared across cores. Being merely a hint, a stale TRUE costs @@ -367,8 +420,41 @@ core_mount(core_handle *spl, platform_status core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns); +/* + * Make every update that linearized before this call recoverable after power + * loss. Writers run concurrently with the in-memory log-group cut, page + * graduation, writeback, and the device barrier. + */ +platform_status +core_durable_barrier(core_handle *spl); + +/* + * Unmount the database without destroying it; it can be re-opened later with + * core_mount(). + * + * An unmount is a sync followed by a shutdown, so it returns an error if it + * cannot guarantee that everything inserted before the call is recoverable. + * Records get there by one of two routes -- folded into the durable trunk root + * by a checkpoint, or held in a durable log -- and which routes are open + * depends on whether every memtable managed to incorporate: + * + * all incorporated: either route carries everything, so the root alone is + * enough and the logs can be discarded. + * some unincorporated: the root will not contain those records, so only a + * durable log can carry them, and it must be preserved for + * replay rather than discarded. + * + * When neither route is available and force is FALSE, the unmount is abandoned + * before destructive teardown: the instance remains mounted and usable, and + * the caller can retry or investigate. Any non-OK result has that meaning. + * + * With force, teardown always completes. STATUS_OK still guarantees that all + * acknowledged data is recoverable; a non-OK result means preservation could + * not be guaranteed. Whether the next mount needs log replay or an allocator + * rebuild is deliberately not part of this return value. + */ platform_status -core_unmount(core_handle *spl); +core_unmount(core_handle *spl, bool32 force); /* Unmount the database and erase it from the disk */ void @@ -432,6 +518,7 @@ core_config_init(core_config *trunk_cfg, uint64 prefetch_budget, bool32 use_log, uint64 checkpoint_log_size_bytes, + uint64 checkpoint_log_grace_bytes, bool32 use_stats, bool32 verbose_logging, platform_log_handle *log_handle); diff --git a/src/data_internal.h b/src/data_internal.h index 09855863..86b05ddd 100644 --- a/src/data_internal.h +++ b/src/data_internal.h @@ -385,6 +385,14 @@ message_materialized_length(message msg) } } +/* Inline messages are covered by their containing page or log record. */ +static inline platform_status +message_validate(message msg) +{ + return message_is_blob(msg) ? blob_validate(msg.cc, message_slice(msg)) + : STATUS_OK; +} + static inline platform_status message_materialize(message msg, merge_accumulator *tmp); diff --git a/src/log.h b/src/log.h index 09ce8afe..daa48868 100644 --- a/src/log.h +++ b/src/log.h @@ -12,63 +12,140 @@ #include "cache.h" #include "data_internal.h" #include "iterator.h" +#include "log_data.h" typedef struct log_handle log_handle; typedef struct log_iterator log_iterator; typedef struct log_config log_config; /* - * The on-disk head of one mini-allocator-backed log stream: the data head - * (where replay begins), the metadata head (which owns the stream's extents), - * and a per-stream magic that validates its pages. Fixed at creation; a - * higher-level checkpoint record stores it to later find the stream for replay - * or reclaim it via log_dec_ref(). + * In order to support high concurrency while ensuring that the log makes + * updates durable in their linearization order, log writes are performed in two + * steps. First, at the linearization point of an update, the caller uses + * log_write_reserve() to reserve a spot in the log. The log_write_token is + * the reservation receipt. Then, they use log_write_reserved() to actually + * write the log entry. We separate the process into two steps because, in order + * to ensure correct linearization ordering of log durability, callers may need + * to reserve their slot in the log while holding locks on other data structures + * that they are updating (e.g. the btree leaf of the memtable). The actual + * write, which may require performing I/O, memory allocation, etc, can occur + * later, outside of any critical section. + * + * There is no way to cancel a reservation, so make the reservation only once + * you know that you want to perform the write. + * + * If the write fails, then the log will not satisfy subsequent make_durable + * calls. */ -typedef struct log_head { - uint64 addr; // data head: first log page, where replay begins - uint64 meta_addr; // mini-allocator metadata head; owns the stream's extents - uint64 magic; // per-stream magic; validates the stream's pages -} log_head; - -typedef int (*log_write_fn)(log_handle *log, - key tuple_key, - message data, - uint64 memtable_generation, - uint64 leaf_generation); +typedef struct log_write_token { + log_handle *log; + void *internal; + /* + * Reservations are thread-affine and may not be nested. The concrete log + * records both the originating thread and its reservation ticket here so + * write_reserved() can validate the receipt before consuming it. + */ + threadid owner_tid; + uint64 internal_ticket; +} log_write_token; + +typedef void (*log_write_reserve_fn)(log_handle *log, log_write_token *token); + +/* Append through, and always consume, a prior reservation. */ +typedef int (*log_write_reserved_fn)(log_write_token *token, + key tuple_key, + message data, + uint64 memtable_generation, + uint64 leaf_generation); + /* - * Finalize and retire the log stream, terminally. Finalizes the current - * append pages into checksummed, immutable pages, releases in-memory - * resources, and frees the handle (which is invalid afterward). + * make_durable_{begin,wait}() are used to ensure that all log writes whose + * _reservation_ _completed_ before the _beginning_ of make_durable_begin() will + * be durable before the _end_ of make_durable_wait(). * - * The caller must exclude concurrent log_write() and log_seal() calls. seal() - * itself issues no writeback or durable barrier: to make the sealed pages - * durable, the caller takes the cache writeback fence + a durable barrier - * afterward. The stream's head is fixed at creation and obtained then via - * log_get_head(), so seal needs no out-parameter; the caller frees the on-disk - * extents later via log_dec_ref(). + * Opaque durability cut returned by log_make_durable_begin(). A successful + * begin holds one reference on the in-memory log handle until the matching + * log_make_durable_wait(). Once the owner has otherwise quiesced and retired + * the stream, that pin lets it deinit the stream between the two calls. + * + * The log_durable_ticket identifies the set of writes covered by the + * make_durable_begin request. (i.e. all writes whose reservation completed + * before the beginning of make_durable_begin()) + * + * If a covered write fails, then make_durable_wait will return an error -- the + * log can never ensure that all covered writes have been made durable. + * + * A thread may not call make_durable_begin() while it owns a write reservation + * on this log. Writers, other make_durable_begin() calls, and log_seal() may + * run concurrently, subject to log_seal()'s exclusion of new reservations. + */ +typedef uint64 log_durable_ticket; + +typedef platform_status ( + *log_make_durable_begin_fn)(log_handle *log, log_durable_ticket *ticket_out); +typedef platform_status (*log_make_durable_wait_fn)(log_handle *log, + log_durable_ticket ticket); + +/* + * Finish the log: ensure that everything in the log (including + * reserved-but-not-yet-written items) is durably written to disk and mark the + * last of it as the end of the log so that replay can tell a complete log from + * one that a crash truncated. The log is immutable afterward, but the handle + * remains valid and must still be released with log_deinit(). + * + * The caller must exclude concurrent execution of log_write_reserve() and + * log_seal(). Tokens returned by earlier reservations may remain outstanding + * and are included in the sealed stream, but the sealing thread must not itself + * own one because seal waits for all such reservations to complete. + * + * A caller that is about to discard the log outright can skip this and + * call log_deinit() alone. + * + * On a transient failure nothing new is guaranteed durable, but seal may + * simply be called again. Note, however, that an earlier failure in a log_write + * means that the log is corrupted (from the point of that write onward) and + * hence can never sealed. + * + * Calling seal again after success is idempotent. */ typedef platform_status (*log_seal_fn)(log_handle *log); + /* - * The stream's durable head, fixed at creation. The caller records it - * (e.g. in the superblock) as soon as the log is created, so that a crash - * mid-stream can find the stream for replay. + * Release the stream owner's reference. Before calling this, the owner must + * exclude concurrent reservations, log_make_durable_begin(), and log_seal() + * calls, including waiting for any already executing calls to return. It need + * not wait for log_make_durable_wait() calls consuming tickets issued before + * deinit. The handle is otherwise invalid as soon as deinit is called. Deinit + * writes nothing, so it cannot fail. + */ +typedef void (*log_deinit_fn)(log_handle *log); + +/* + * The log's head, fixed at creation. The caller records it (e.g. in the + * superblock) so that crash recovery can find the log for replay. */ typedef log_head (*log_head_fn)(log_handle *log); + /* - * Bytes appended to the stream so far, so a caller can decide when to retire - * it. Excludes the implementation's fixed per-stream overhead: a stream that - * has had nothing written to it reports 0, which keeps a size-triggered policy - * from firing on a brand-new stream no matter how small its threshold. A - * conservative measure otherwise -- space is counted as it is reserved, so this - * rounds up to whatever allocation unit the implementation uses. + * Whether the log has ever accepted a record. + */ +typedef bool32 (*log_is_empty_fn)(log_handle *log); + +/* + * Rough approximation of the log's current on-disk size. */ typedef uint64 (*log_size_fn)(log_handle *log); typedef struct log_ops { - log_write_fn write; - log_seal_fn seal; - log_head_fn head; - log_size_fn size; + log_write_reserve_fn write_reserve; + log_write_reserved_fn write_reserved; + log_make_durable_begin_fn make_durable_begin; + log_make_durable_wait_fn make_durable_wait; + log_seal_fn seal; + log_deinit_fn deinit; + log_head_fn head; + log_is_empty_fn is_empty; + log_size_fn size; } log_ops; // to sub-class log, make a log_handle your first field @@ -76,6 +153,30 @@ struct log_handle { const log_ops *ops; }; +static inline void +log_write_reserve(log_handle *log, log_write_token *token) +{ + platform_assert(log != NULL); + platform_assert(token != NULL); + log->ops->write_reserve(log, token); +} + +/* Append a reserved record and consume token on every return path. */ +static inline int +log_write_reserved(log_write_token *token, + key tuple_key, + message data, + uint64 memtable_generation, + uint64 leaf_generation) +{ + platform_assert(token != NULL); + platform_assert(token->log != NULL, + "log write token is absent or has already been consumed"); + return token->log->ops->write_reserved( + token, tuple_key, data, memtable_generation, leaf_generation); +} + +/* Convenience for callers whose reservation and append are adjacent. */ static inline int log_write(log_handle *log, key tuple_key, @@ -83,15 +184,55 @@ log_write(log_handle *log, uint64 memtable_generation, uint64 leaf_generation) { - return log->ops->write( - log, tuple_key, data, memtable_generation, leaf_generation); + log_write_token token; + log_write_reserve(log, &token); + return log_write_reserved( + &token, tuple_key, data, memtable_generation, leaf_generation); } /* - * Finalize and retire the log, freeing the handle. See log_seal_fn for the - * required exclusion and durability ordering; the handle is invalid after this - * returns. Capture the head via log_get_head() beforehand (it is fixed at - * creation). + * Take a quick cut of everything reserved for write so far and return a ticket + * for it. Writers may run concurrently with this operation. A successful begin + * must be paired with exactly one log_make_durable_wait(), even when ticket_out + * is zero or a later operation makes the cut durable first. + * + * The ticket prevents the handle from being freed until wait consumes it. This + * permits a caller to release whatever external lock protects the live-log + * pointer before doing the slow wait. After separately excluding every new + * operation as required by log_deinit_fn, the owner may also deinit the stream + * before this wait. + */ +static inline platform_status +log_make_durable_begin(log_handle *log, log_durable_ticket *ticket_out) +{ + platform_assert(log != NULL); + platform_assert(ticket_out != NULL); + return log->ops->make_durable_begin(log, ticket_out); +} + +/* Wait for the cut and consume the handle pin acquired by begin. */ +static inline platform_status +log_make_durable_wait(log_handle *log, log_durable_ticket ticket) +{ + return log->ops->make_durable_wait(log, ticket); +} + +/* Convenience wrapper for callers that do not need to release a lock early. */ +static inline platform_status +log_make_durable(log_handle *log) +{ + log_durable_ticket ticket; + platform_status rc = log_make_durable_begin(log, &ticket); + if (!SUCCESS(rc)) { + return rc; + } + return log_make_durable_wait(log, ticket); +} + +/* + * Finish the log, durably. See log_seal_fn for the required exclusion. The + * handle stays valid; release it with log_deinit(). Capture the head via + * log_get_head() beforehand (it is fixed at creation). */ static inline platform_status log_seal(log_handle *log) @@ -99,6 +240,16 @@ log_seal(log_handle *log) return log->ops->seal(log); } +/* + * Release the quiesced owner; only already-issued ticket waits remain legal + * afterward. See log_deinit_fn for the required exclusion. + */ +static inline void +log_deinit(log_handle *log) +{ + log->ops->deinit(log); +} + /* The stream's durable head (fixed at creation). See log_head_fn. */ static inline log_head log_get_head(log_handle *log) @@ -106,6 +257,13 @@ log_get_head(log_handle *log) return log->ops->head(log); } +/* Whether this stream has accepted any records. */ +static inline bool32 +log_is_empty(log_handle *log) +{ + return log->ops->is_empty(log); +} + /* Bytes the stream currently occupies on disk. See log_size_fn. */ static inline uint64 log_get_size(log_handle *log) @@ -116,23 +274,14 @@ log_get_size(log_handle *log) /* * A log_handle is created by the concrete log implementation -- e.g. * shard_log_create() -- and then driven through the abstract ops above; it is - * freed by log_seal(). - */ - -/* - * Release a sealed log identified by its log_head: drop the reference its - * metadata head holds, freeing the stream's on-disk extents. Takes no handle - * -- the handle was freed by log_seal(); the caller retained only the head - * (log_get_head(), captured at creation). + * freed by log_deinit(). */ -void -log_dec_ref(cache *cc, const log_head *head); /* * ---- Abstract log iteration ---- * - * A log_iterator reads a sealed log's records in generation order (used by - * crash recovery to replay a stream onto the durable root). It is a generic + * A log_iterator reads a log's records in generation order (used by + * crash recovery to replay a log onto the durable root). It is a generic * iterator (curr/can_next/next, via the embedded `super`) plus the log-specific * ops below. To sub-class, make a log_iterator your first field. */ @@ -140,10 +289,12 @@ typedef void (*log_iterator_curr_generations_fn)(log_iterator *itor, uint64 *memtable_generation, uint64 *leaf_generation); typedef void (*log_iterator_deinit_fn)(log_iterator *itor); +typedef bool32 (*log_iterator_stream_complete_fn)(log_iterator *itor); typedef struct log_iterator_ops { log_iterator_curr_generations_fn curr_generations; log_iterator_deinit_fn deinit; + log_iterator_stream_complete_fn stream_complete; } log_iterator_ops; struct log_iterator { @@ -188,6 +339,22 @@ log_iterator_next(log_iterator *itor) return iterator_next(&itor->super); } +/* + * Whether the records this iterator yields run all the way to the end of a + * sealed log, as opposed to stopping early because the log was truncated + * by a crash. + * + * Recovery needs this to decide whether it may go on to the next log. The + * records of a truncated log, log_A, are still a valid prefix on their own, but + * anything written to a subsequent log, log_B, must not be replayed on top of + * them: doing so would skip whatever was lost at the end of log_A. + */ +static inline bool32 +log_iterator_stream_complete(log_iterator *itor) +{ + return itor->ops->stream_complete(itor); +} + /* Free the iterator and its resources; the handle is invalid afterward. */ static inline void log_iterator_deinit(log_iterator *itor) diff --git a/src/log_data.h b/src/log_data.h new file mode 100644 index 00000000..a3a8a2f6 --- /dev/null +++ b/src/log_data.h @@ -0,0 +1,49 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* Shared, disk-resident log identity types. */ + +#pragma once + +#include + +#include "splinterdb/platform_linux/public_platform.h" + +typedef struct log_nonce { + uint64 high; + uint64 low; +} log_nonce; + +_Static_assert(sizeof(log_nonce) == 16, "on-disk log nonce layout changed"); + +/* + * The on-disk head of one mini-allocator-backed log stream: the data head + * (where replay begins), the metadata head (which owns the stream's extents), + * and a per-stream nonce that validates its pages. Fixed at creation and + * shared by the log interface and higher-level durable records. + */ +typedef struct log_head { + uint64 addr; + uint64 meta_addr; + log_nonce nonce; +} log_head; + +_Static_assert(offsetof(log_head, addr) == 0, + "log data address layout changed"); +_Static_assert(offsetof(log_head, meta_addr) == 8, + "log metadata address layout changed"); +_Static_assert(offsetof(log_head, nonce) == 16, "log nonce offset changed"); +_Static_assert(sizeof(log_head) == 32, "on-disk log head layout changed"); + +static inline bool32 +log_nonce_is_equal(log_nonce left, log_nonce right) +{ + return left.high == right.high && left.low == right.low; +} + +static inline bool32 +log_head_is_equal(log_head left, log_head right) +{ + return left.addr == right.addr && left.meta_addr == right.meta_addr + && log_nonce_is_equal(left.nonce, right.nonce); +} diff --git a/src/memtable.c b/src/memtable.c index 13793514..06cc863b 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -68,7 +68,7 @@ memtable_process(memtable_context *ctxt, uint64 generation) ctxt->process(ctxt->process_ctxt, generation); } -static inline void +void memtable_begin_insert(memtable_context *ctxt) { batch_rwlock_get(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); @@ -118,6 +118,29 @@ memtable_end_insert_rotation(memtable_context *ctxt) batch_rwlock_unclaim(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } +/* + * A rotation may advance to the next generation only once its ring slot has + * been recycled. Both natural and forced rotations test this while inserts + * are excluded from changing the active generation. + */ +static inline platform_status +memtable_next_generation_status(memtable_context *ctxt, + uint64 current_generation) +{ + uint64 next_generation = current_generation + 1; + uint64 next_mt_no = next_generation % ctxt->cfg.max_memtables; + memtable *next_mt = &ctxt->mt[next_mt_no]; + + if (next_mt->state == MEMTABLE_STATE_READY) { + return STATUS_OK; + } + if (next_mt->state == MEMTABLE_STATE_INCORPORATION_FAILED) { + platform_assert(!SUCCESS(next_mt->incorporation_status)); + return next_mt->incorporation_status; + } + return STATUS_BUSY; +} + void memtable_begin_lookup(memtable_context *ctxt) { @@ -167,12 +190,11 @@ memtable_maybe_rotate_and_begin_insert(memtable_context *ctxt, if (memtable_is_full(&ctxt->cfg, current_mt)) { // If the current memtable is full, try to retire it - uint64 next_generation = current_generation + 1; - uint64 next_mt_no = next_generation % ctxt->cfg.max_memtables; - memtable *next_mt = &ctxt->mt[next_mt_no]; - if (next_mt->state != MEMTABLE_STATE_READY) { + platform_status rotation_rc = + memtable_next_generation_status(ctxt, current_generation); + if (!SUCCESS(rotation_rc)) { memtable_end_insert(ctxt); - return STATUS_BUSY; + return rotation_rc; } if (memtable_try_begin_insert_rotation(ctxt)) { @@ -309,14 +331,35 @@ memtable_mark_incorporation_failed(memtable *mt, platform_status status) MEMTABLE_STATE_INCORPORATION_FAILED); } -uint64 -memtable_force_rotation(memtable_context *ctxt) +platform_status +memtable_force_rotation_if(memtable_context *ctxt, + memtable_rotation_predicate_fn predicate, + void *predicate_arg, + uint64 *generation_out) { + platform_assert(predicate != NULL); + memtable_block_inserts(ctxt); - uint64 generation = ctxt->generation; - uint64 mt_no = generation % ctxt->cfg.max_memtables; - memtable *mt = &ctxt->mt[mt_no]; + /* + * Revalidate before the ring-readiness check so a stale request is a clean + * no-op even when the next ring slot is busy. The predicate is observation- + * only: a later readiness failure must leave its higher-level state pending. + */ + if (!predicate(predicate_arg)) { + memtable_unblock_inserts(ctxt); + return STATUS_OK; + } + + uint64 generation = ctxt->generation; + platform_status rc = memtable_next_generation_status(ctxt, generation); + if (!SUCCESS(rc)) { + memtable_unblock_inserts(ctxt); + return rc; + } + + uint64 mt_no = generation % ctxt->cfg.max_memtables; + memtable *mt = &ctxt->mt[mt_no]; memtable_transition(mt, MEMTABLE_STATE_READY, MEMTABLE_STATE_FINALIZED); uint64 current_generation = ctxt->generation++; platform_assert(ctxt->generation - ctxt->generation_retired @@ -339,7 +382,24 @@ memtable_force_rotation(memtable_context *ctxt) // the natural rotation path does. memtable_process(ctxt, current_generation); - return current_generation; + if (generation_out != NULL) { + *generation_out = current_generation; + } + return STATUS_OK; +} + +static bool32 +memtable_rotation_always_requested(void *arg) +{ + (void)arg; + return TRUE; +} + +platform_status +memtable_force_rotation(memtable_context *ctxt, uint64 *generation_out) +{ + return memtable_force_rotation_if( + ctxt, memtable_rotation_always_requested, NULL, generation_out); } void diff --git a/src/memtable.h b/src/memtable.h index 443cd5cf..68798edd 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -112,6 +112,16 @@ memtable_transition(memtable *mt, typedef void (*process_fn)(void *arg, uint64 generation); +/* + * Called with inserts excluded to decide whether a requested forced rotation + * is still needed. The callback must remain I/O-free, must not acquire a + * memtable lock, and must not mutate the memtable context; lock-free inspection + * such as memtable_generation() is permitted. It is an observation-only + * predicate: it must not claim or otherwise mutate higher-level state, because + * a later ring-readiness check can still prevent the requested rotation. + */ +typedef bool32 (*memtable_rotation_predicate_fn)(void *arg); + typedef struct memtable_config { uint64 max_extents_per_memtable; uint64 max_memtables; @@ -167,6 +177,15 @@ platform_status memtable_maybe_rotate_and_begin_insert(memtable_context *ctxt, uint64 *generation); +/* + * Pin the current insert generation without excluding other inserts. This is + * useful for operations which need the same lifetime protection as an insert + * but do not themselves select or mutate a memtable. Pair with + * memtable_end_insert(). + */ +void +memtable_begin_insert(memtable_context *ctxt); + void memtable_end_insert(memtable_context *ctxt); @@ -205,19 +224,54 @@ memtable_mark_incorporation_failed(memtable *mt, platform_status status); const char * memtable_state_string(memtable_state state); +/* + * Conditionally rotate the current memtable, regardless of fullness. + * + * This function first excludes inserts and invokes predicate(arg). If the + * predicate returns FALSE, no memtable state is changed and the function + * returns STATUS_OK, leaving generation_out unchanged. This revalidation + * prevents a stale higher-level request from causing a second rotation after a + * natural rotation already satisfied it. + * + * If the predicate returns TRUE, the function next checks whether the following + * memtable-ring slot is ready. It returns STATUS_BUSY without changing the + * context if work on that slot is still in progress, or propagates the slot's + * recorded failure if incorporation failed. Because this check can fail after + * predicate returns TRUE, the predicate must not claim state. + * + * Once the ring slot is ready, this performs the same sequence as a natural + * (fullness-triggered) rotation: finalize the memtable, advance the generation, + * invoke the rotate callback under insert exclusion, then -- once inserts are + * unblocked -- invoke the process callback to dispatch the rotated memtable. + * If it rotates, the finalized generation is written to generation_out when + * non-NULL. + */ +platform_status +memtable_force_rotation_if(memtable_context *ctxt, + memtable_rotation_predicate_fn predicate, + void *predicate_arg, + uint64 *generation_out); + /* * Rotate the current memtable now, regardless of fullness. Performs the same * sequence as the natural (fullness-triggered) rotation: finalize the memtable, * advance the generation, invoke the rotate callback under insert exclusion, * then -- once inserts are unblocked -- invoke the process callback to dispatch - * the rotated memtable. Returns the finalized generation. + * the rotated memtable. * - * Callers therefore need do nothing further: whatever the rotate callback - * started (e.g. core's checkpoint log cut) is resolved by the process callback, - * just as it is for a natural rotation. + * Returns STATUS_BUSY without changing the context if work on the next + * memtable-ring slot is still in progress, or propagates that slot's recorded + * failure if incorporation failed. This is the same readiness condition that + * prevents a natural rotation from overtaking incorporation. On success, + * writes the finalized generation to generation_out when non-NULL. + * + * This is the unconditional wrapper around memtable_force_rotation_if(). + * Callers therefore need do nothing further after success: whatever the rotate + * callback started (e.g. core's checkpoint log cut) is resolved by the process + * callback, just as it is for a natural rotation. */ -uint64 -memtable_force_rotation(memtable_context *ctxt); +platform_status +memtable_force_rotation(memtable_context *ctxt, uint64 *generation_out); void memtable_init(memtable *mt, cache *cc, memtable_config *cfg, uint64 generation); diff --git a/src/mini_allocator.c b/src/mini_allocator.c index a4f328d6..1a657ef2 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -1189,7 +1189,7 @@ mini_recover_visit_meta_page(cache *cc, * allocator roots, remains the responsibility of the higher-level * trunk/log recovery walker. */ -platform_status +static platform_status mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type) { if (cc == NULL || meta_type < PAGE_TYPE_FIRST || meta_type >= NUM_PAGE_TYPES) @@ -1224,6 +1224,28 @@ mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type) &state); } +platform_status +mini_recover_references(cache *cc, uint64 meta_head, page_type type) +{ + allocator *al = cache_get_allocator(cc); + uint64 base = base_addr(cc, meta_head); + + if (allocator_get_refcount(al, base) == AL_FREE) { + platform_status rc = mini_recover_allocations(cc, meta_head, type); + if (!SUCCESS(rc)) { + return rc; + } + } + /* + * Every holder adds one, the first one included. Enumerating the extents + * accounts for the mini allocator existing -- it leaves the metadata extent + * at MINI_NO_REFS, which is what mini_init_with_types() establishes with its + * "meta_page gets an extra ref" -- and says nothing about who refers to it. + * External references are the count above that, so each is recorded here. + */ + return allocator_recovery_record_reference(al, base, type); +} + /* *----------------------------------------------------------------------------- * mini_meta_cursor -- cursor over a mini_allocator's extent entries. diff --git a/src/mini_allocator.h b/src/mini_allocator.h index 5407530f..5983936f 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -114,31 +114,30 @@ void mini_prefetch(cache *cc, page_type type, uint64 meta_head); /* - * mini_recover_allocations -- + * mini_recover_references -- * - * Rebuild allocator references for a finalized mini allocator without - * trusting allocator refcounts. This is the discovery primitive crash - * recovery uses to reconstruct them: it records one allocator reference - * (via allocator_recovery_record_reference()) for every metadata extent - * and for every data-extent entry in the on-disk mini metadata stream. - * Data entries are deliberately not deduplicated: repeated entries - * represent repeated references in the mini allocator and are recorded - * exactly as often as they occur. meta_type is the page type of - * meta_head's own chain; each data extent's type is read from its own - * metadata entry, so it is not a parameter here. + * Record one logical reference to this mini allocator during a crash- + * recovery rebuild, enumerating its extents first if nothing had reached + * it yet. The recovery counterpart of mini_inc_ref(). * - * Validates the metadata-page chain, page-header bounds, page types, - * batches, and extent addresses before using them. A malformed on-disk - * stream returns STATUS_INVALID_STATE. This function performs no writes - * and does not use allocator refcounts to decide what to traverse. + * The refcount map under construction doubles as the set of mini + * allocators already enumerated, which is what saves callers from keeping + * a visited set of their own: a free metadata extent means this is the + * first reference and the interior is still unknown, so + * mini_recover_allocations() runs; a referenced one means some earlier + * caller already enumerated it and all this adds is multiplicity. + * Sharing is expected -- one branch commonly lives in many trunk nodes. * - * This is a physical enumeration only. Recovering logical reference - * multiplicity, and deduplicating references shared by distinct mini - * allocator roots, remains the responsibility of the higher-level - * trunk/log recovery walker. + * The multiplicity it produces matches normal operation, because a logical + * reference is exactly one allocator reference on the extent holding the + * metadata head, as mini_inc_ref() shows. That reference is recorded for + * every holder, the first one included: enumerating the extents leaves the + * metadata extent at MINI_NO_REFS, which accounts for the mini allocator + * existing rather than for anyone referring to it, so external references + * are counted on top of it. */ platform_status -mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type); +mini_recover_references(cache *cc, uint64 meta_head, page_type type); /* * mini_meta_cursor: a non-blocking cursor over the extent entries of a diff --git a/src/platform_linux/laio.c b/src/platform_linux/laio.c index 6988514b..3d5f59b2 100644 --- a/src/platform_linux/laio.c +++ b/src/platform_linux/laio.c @@ -24,7 +24,9 @@ #include "platform_typed_alloc.h" #include "async.h" #include "platform_log.h" +#include #include +#include #include #include #include @@ -162,6 +164,116 @@ laio_cleaner(void *arg) return NULL; } +/* + * A size cache is invalidated, rather than updated, after writes because writes + * need not be append-only and a short write may still have extended the file. + * The next range query obtains the authoritative logical size from the kernel. + */ +static inline void +laio_invalidate_logical_size(laio_handle *io) +{ + __atomic_fetch_add(&io->write_generation, 1, __ATOMIC_RELEASE); +} + +static platform_status +laio_get_logical_size(laio_handle *io, uint64 *size) +{ + if (io->backing_type == LAIO_BACKING_REGULAR) { + struct stat statbuf; + if (fstat(io->fd, &statbuf) != 0) { + int saved_errno = errno; + platform_error_log("fstat failed while querying logical size: %s\n", + strerror(saved_errno)); + return CONST_STATUS(saved_errno); + } + if (statbuf.st_size < 0) { + platform_error_log("fstat returned negative logical size %ld\n", + (long)statbuf.st_size); + return STATUS_IO_ERROR; + } + *size = (uint64)statbuf.st_size; + return STATUS_OK; + } + + if (io->backing_type == LAIO_BACKING_BLOCK) { + uint64 block_size; + if (ioctl(io->fd, BLKGETSIZE64, &block_size) != 0) { + int saved_errno = errno; + platform_error_log( + "BLKGETSIZE64 failed while querying logical size: %s\n", + strerror(saved_errno)); + return CONST_STATUS(saved_errno); + } + *size = block_size; + return STATUS_OK; + } + + platform_error_log("Cannot query the logical size of this backing object\n"); + return STATUS_NOTSUP; +} + +static platform_status +laio_range_is_readable(io_handle *ioh, + uint64 addr, + uint64 bytes, + bool32 *readable) +{ + laio_handle *io = (laio_handle *)ioh; + + if (readable == NULL) { + return STATUS_BAD_PARAM; + } + *readable = FALSE; + + if (UINT64_MAX - addr < bytes) { + return STATUS_BAD_PARAM; + } + + uint64 logical_size; + while (TRUE) { + uint64 write_generation = + __atomic_load_n(&io->write_generation, __ATOMIC_ACQUIRE); + uint64 size_generation = + __atomic_load_n(&io->logical_size_generation, __ATOMIC_ACQUIRE); + + if (size_generation == write_generation) { + logical_size = __atomic_load_n(&io->logical_size, __ATOMIC_RELAXED); + } else { + platform_status rc = laio_get_logical_size(io, &logical_size); + if (!SUCCESS(rc)) { + return rc; + } + + /* Do not publish a size observed concurrently with a completion. */ + if (__atomic_load_n(&io->write_generation, __ATOMIC_ACQUIRE) + != write_generation) + { + continue; + } + + __atomic_store_n(&io->logical_size, logical_size, __ATOMIC_RELAXED); + __atomic_store_n( + &io->logical_size_generation, write_generation, __ATOMIC_RELEASE); + } + + /* + * A completion after either the cached load or the cache publication + * invalidates the value. Retry so that a query starting after io_write() + * returns, or running from an async completion callback, cannot see the + * pre-write EOF. + */ + if (__atomic_load_n(&io->write_generation, __ATOMIC_ACQUIRE) + != write_generation) + { + continue; + } + break; + } + + *readable = addr <= logical_size && bytes <= logical_size - addr; + return STATUS_OK; +} + /* * laio_read() - Basically a wrapper around pread(). */ @@ -169,24 +281,35 @@ static platform_status laio_read(io_handle *ioh, void *buf, uint64 bytes, uint64 addr) { laio_handle *io; - int ret; + ssize_t ret; - io = (laio_handle *)ioh; - ret = pread(io->fd, buf, bytes, addr); + io = (laio_handle *)ioh; + do { + ret = pread(io->fd, buf, bytes, addr); + } while (ret < 0 && errno == EINTR); #if defined(__has_feature) # if __has_feature(memory_sanitizer) - __msan_unpoison(buf, ret); + if (ret > 0) { + __msan_unpoison(buf, ret); + } # endif #endif - if (ret == bytes) { + if (ret >= 0 && (uint64)ret == bytes) { return STATUS_OK; } - platform_error_log("laio_read: pread failed for addr %lu, bytes %lu, " - "ret %d: %s\n", - addr, - bytes, - ret, - strerror(errno)); + if (ret < 0) { + platform_error_log("laio_read: pread failed for addr %lu, bytes %lu: " + "%s\n", + addr, + bytes, + strerror(errno)); + } else { + platform_error_log("laio_read: short read for addr %lu, bytes %lu, " + "read %ld\n", + addr, + bytes, + (long)ret); + } return STATUS_IO_ERROR; } @@ -196,21 +319,46 @@ laio_read(io_handle *ioh, void *buf, uint64 bytes, uint64 addr) static platform_status laio_write(io_handle *ioh, void *buf, uint64 bytes, uint64 addr) { - laio_handle *io; - int ret; - - io = (laio_handle *)ioh; - ret = pwrite(io->fd, buf, bytes, addr); - if (ret == bytes) { - return STATUS_OK; + laio_handle *io = (laio_handle *)ioh; + char *cursor = buf; + uint64 offset = addr; + uint64 remaining = bytes; + + /* + * Finish a short write rather than reporting it. pwrite() is permitted to + * satisfy only part of the request, and the cache's writeback-retry path + * treats any failure here as a hard I/O error -- so returning early on a + * write that was merely chunked would mark a perfectly good page as failed + * and have it retried forever. + */ + while (remaining > 0) { + ssize_t ret = pwrite(io->fd, cursor, remaining, offset); + if (ret < 0) { + if (errno == EINTR) { + continue; + } + platform_error_log("laio_write: pwrite failed for addr %lu, " + "bytes %lu, remaining %lu: %s\n", + addr, + bytes, + remaining, + strerror(errno)); + return STATUS_IO_ERROR; + } + if (ret == 0) { + platform_error_log("laio_write: pwrite made no progress for addr %lu, " + "bytes %lu, remaining %lu\n", + addr, + bytes, + remaining); + return STATUS_IO_ERROR; + } + laio_invalidate_logical_size(io); + cursor += ret; + offset += ret; + remaining -= ret; } - platform_error_log("laio_write: pwrite failed for addr %lu, bytes %lu, " - "ret %d: %s\n", - addr, - bytes, - ret, - strerror(errno)); - return STATUS_IO_ERROR; + return STATUS_OK; } /* @@ -306,8 +454,7 @@ typedef struct laio_async_state { struct iocb *reqs[1]; int status; uint64 iovlen; - struct iovec *iovs; - struct iovec iov[]; + struct iovec iovs[]; } laio_async_state; _Static_assert( @@ -317,10 +464,6 @@ _Static_assert( static void laio_async_state_deinit(io_async_state *ios) { - laio_async_state *lios = (laio_async_state *)ios; - if (lios->iovs != lios->iov) { - platform_free(PROCESS_PRIVATE_HEAP_ID, lios->iovs); - } } static platform_status @@ -359,6 +502,15 @@ laio_async_callback(io_context_t ctx, struct iocb *iocb, long res, long res2) laio_async_state *ios = (laio_async_state *)((char *)iocb - offsetof(laio_async_state, req)); ios->status = res; + /* + * A positive short completion may have extended a regular file. Invalidate + * for every write completion, including failures, because the completion + * result alone does not have to prove that no bytes reached the device. + * Publish the invalidation before the client callback may issue a query. + */ + if (ios->cmd == io_async_pwritev) { + laio_invalidate_logical_size(ios->io); + } if (ios->callback) { ios->callback(ios->callback_arg); } @@ -544,25 +696,8 @@ laio_async_state_init(io_async_state *state, async_callback_fn callback, void *callback_arg) { - laio_async_state *ios = (laio_async_state *)state; - laio_handle *io = (laio_handle *)gio; - uint64 pages_per_extent = io->cfg->extent_size / io->cfg->page_size; - - if (sizeof(*ios) + pages_per_extent * sizeof(struct iovec) - <= IO_ASYNC_STATE_BUFFER_SIZE) - { - ios->iovs = ios->iov; - } else { - ios->iovs = TYPED_ARRAY_MALLOC( - PROCESS_PRIVATE_HEAP_ID, ios->iovs, pages_per_extent); - if (ios->iovs == NULL) { - platform_error_log("laio_async_state_init: failed to allocate iovec " - "array for addr %lu, pages_per_extent %lu\n", - addr, - pages_per_extent); - return STATUS_NO_MEMORY; - } - } + laio_async_state *ios = (laio_async_state *)state; + laio_handle *io = (laio_handle *)gio; ios->super.ops = &laio_async_state_ops; ios->__async_state_stack[0] = ASYNC_STATE_INIT; @@ -698,14 +833,15 @@ laio_process_termination_callback(threadid pid, void *arg) * Define an implementation of the abstract IO Ops interface methods. */ static io_ops laio_ops = { - .read = laio_read, - .write = laio_write, - .async_state_init = laio_async_state_init, - .cleanup = laio_cleanup, - .wait_all = laio_wait_all, - .durable_barrier = laio_durable_barrier, - .print_stats = laio_print_stats, - .reset_stats = laio_reset_stats, + .read = laio_read, + .write = laio_write, + .range_is_readable = laio_range_is_readable, + .async_state_init = laio_async_state_init, + .cleanup = laio_cleanup, + .wait_all = laio_wait_all, + .durable_barrier = laio_durable_barrier, + .print_stats = laio_print_stats, + .reset_stats = laio_reset_stats, }; /* @@ -756,6 +892,16 @@ laio_handle_create(io_config *cfg, platform_heap_id hid) return NULL; } + if (S_ISREG(statbuf.st_mode)) { + io->backing_type = LAIO_BACKING_REGULAR; + } else if (S_ISBLK(statbuf.st_mode)) { + io->backing_type = LAIO_BACKING_BLOCK; + } else { + io->backing_type = LAIO_BACKING_UNSUPPORTED; + } + /* Generation zero is reserved for an invalid, never-populated size cache. */ + io->write_generation = 1; + // 32 4KB pages #define EXTENT_SIZE (32 * 4 * 1024) @@ -879,11 +1025,20 @@ laio_config_valid(io_config *cfg) cfg->page_size); return STATUS_BAD_PARAM; } + if (!laio_config_valid_extent_size(cfg)) { platform_error_log( "Extent-size, %lu bytes, is an invalid IO configuration.\n", cfg->extent_size); return STATUS_BAD_PARAM; } + uint64 pages_per_extent = cfg->extent_size / cfg->page_size; + + if (IO_ASYNC_STATE_BUFFER_SIZE + < sizeof(laio_async_state) + pages_per_extent * sizeof(struct iovec)) + { + return STATUS_BAD_PARAM; + } + return STATUS_OK; } diff --git a/src/platform_linux/laio.h b/src/platform_linux/laio.h index ae97acd6..d48876ce 100644 --- a/src/platform_linux/laio.h +++ b/src/platform_linux/laio.h @@ -21,6 +21,12 @@ typedef enum process_context_state { PROCESS_CONTEXT_STATE_SHUTTING_DOWN, } process_context_state; +typedef enum laio_backing_type { + LAIO_BACKING_REGULAR, + LAIO_BACKING_BLOCK, + LAIO_BACKING_UNSUPPORTED, +} laio_backing_type; + #define LAIO_QD_HIST_BUCKETS (IO_DEFAULT_KERNEL_QUEUE_SIZE + 2) typedef struct io_process_context { @@ -45,6 +51,19 @@ typedef struct laio_handle { io_process_context ctx[MAX_THREADS]; platform_heap_id heap_id; int fd; // File descriptor to Splinter device/file. + laio_backing_type backing_type; + + /* + * Cached logical backing size. A completed write advances + * write_generation; range queries may reuse logical_size only when its + * generation matches. The fields are accessed with __atomic builtins -- + * keeping writes to one atomic increment and avoiding a mutex on the IO hot + * path. + */ + uint64 logical_size; + uint64 logical_size_generation; + uint64 write_generation; + process_event_callback_list_node pecnode; } laio_handle; diff --git a/src/platform_linux/platform.h b/src/platform_linux/platform.h index 812ccc44..bd9bb294 100644 --- a/src/platform_linux/platform.h +++ b/src/platform_linux/platform.h @@ -27,4 +27,5 @@ #include "platform_typed_alloc.h" #include "platform_sleep.h" #include "platform_hash.h" -#include "platform_spinlock.h" \ No newline at end of file +#include "platform_random.h" +#include "platform_spinlock.h" diff --git a/src/platform_linux/platform_io.h b/src/platform_linux/platform_io.h index cb60e861..463d7581 100644 --- a/src/platform_linux/platform_io.h +++ b/src/platform_linux/platform_io.h @@ -65,6 +65,10 @@ typedef platform_status (*io_write_fn)(io_handle *io, void *buf, uint64 bytes, uint64 addr); +typedef platform_status (*io_range_is_readable_fn)(io_handle *io, + uint64 addr, + uint64 bytes, + bool32 *readable); #define IO_ASYNC_STATE_BUFFER_SIZE (1024) typedef uint8 io_async_state_buffer[IO_ASYNC_STATE_BUFFER_SIZE]; @@ -93,6 +97,7 @@ typedef void *(*io_get_context_fn)(io_handle *io); typedef struct io_ops { io_read_fn read; io_write_fn write; + io_range_is_readable_fn range_is_readable; io_async_state_init_fn async_state_init; io_cleanup_fn cleanup; io_wait_all_fn wait_all; @@ -139,6 +144,29 @@ io_read(io_handle *io, void *buf, uint64 bytes, uint64 addr) return io->ops->read(io, buf, bytes, addr); } +/* + * Report whether the complete half-open range [addr, addr + bytes) is + * currently readable from the backing object. This is a logical-size query, + * not a promise that the range contains allocated blocks, was ever written, or + * is durable: a sparse hole below a regular file's EOF is readable and reports + * TRUE. Callers must still validate the data they subsequently read. + * Regular files are bounded by their current logical EOF; block devices are + * bounded by the capacity reported by the kernel. + * + * Query errors are returned separately from absence. On success, a zero-byte + * range is readable exactly when addr is no greater than the logical size. + * An overflowing range is invalid and returns STATUS_BAD_PARAM. + * + * The result is coherent with writes made through this io_handle. As with the + * rest of the IO interface, modifying the backing object independently is not + * supported. + */ +static inline platform_status +io_range_is_readable(io_handle *io, uint64 addr, uint64 bytes, bool32 *readable) +{ + return io->ops->range_is_readable(io, addr, bytes, readable); +} + static inline platform_status io_write(io_handle *io, void *buf, uint64 bytes, uint64 addr) { diff --git a/src/platform_linux/platform_random.h b/src/platform_linux/platform_random.h new file mode 100644 index 00000000..e65ba2d0 --- /dev/null +++ b/src/platform_linux/platform_random.h @@ -0,0 +1,40 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * platform_random.h -- + * + * Operating-system entropy for persistent random identifiers. + */ + +#pragma once + +#include "platform_status.h" + +#include +#include +#include + +/* + * Fill buf from the kernel CSPRNG. Short reads and EINTR are normal and are + * handled here so callers either receive every requested byte or an error. + */ +static inline platform_status +platform_random_bytes(void *buf, size_t length) +{ + char *cursor = buf; + + while (length != 0) { + ssize_t got = getrandom(cursor, length, 0); + if (got < 0 && errno == EINTR) { + continue; + } + if (got <= 0) { + return STATUS_IO_ERROR; + } + cursor += got; + length -= got; + } + + return STATUS_OK; +} diff --git a/src/rc_allocator.c b/src/rc_allocator.c index 2bedb5c0..d1f04417 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -145,6 +145,21 @@ rc_allocator_init_refcounts(rc_allocator *al) } memset(al->ref_count, 0, rc_allocator_refcount_buffer_size(al->cfg)); + /* + * Set, not accumulated, for the same reason rc_allocator_load_refcounts() + * does it: this establishes a map rather than adding to one, and a rebuild + * may run against an allocator whose stats are not freshly zeroed. The + * reserved extents counted just below are then the whole of it. + * + * The per-type histograms go with it. A rebuild re-derives the map from + * what is on disk, so the allocations it records are the same ones an + * earlier round recorded, not new ones; leaving them to accumulate would + * make rc_allocator_assert_noleaks() report every rebuilt extent as leaked, + * since nothing will ever deallocate it twice. + */ + al->stats.curr_allocated = 0; + memset(al->stats.extent_allocs, 0, sizeof(al->stats.extent_allocs)); + memset(al->stats.extent_deallocs, 0, sizeof(al->stats.extent_deallocs)); /* * Extent 0 holds the superblock; the refcount map begins at extent 1. @@ -234,6 +249,16 @@ rc_allocator_valid_config(allocator_config *cfg) platform_status rc_allocator_recovery_begin(rc_allocator *al) { + /* + * Cleared before the map is rebuilt, which is what makes a second rebuild + * possible on an allocator that already finished one: while it is set, + * rc_allocator_recovery_record_reference() refuses and + * rc_allocator_recovery_finish() asserts. Recovery rebuilds twice -- once + * including the logs so replay can run without being handed their space, and + * again from the root alone once replay is done, which is what releases that + * space. + */ + al->map_is_valid = FALSE; return rc_allocator_init_refcounts(al); } @@ -397,6 +422,7 @@ rc_allocator_persist(rc_allocator *al, uint64 *state_addr) refcount rc_allocator_inc_ref(rc_allocator *al, uint64 addr) { + platform_assert(al->map_is_valid); debug_assert(rc_allocator_valid_extent_addr(al, addr)); uint64 extent_no = addr / al->cfg->io_cfg->extent_size; @@ -416,6 +442,7 @@ rc_allocator_inc_ref(rc_allocator *al, uint64 addr) refcount rc_allocator_dec_ref(rc_allocator *al, uint64 addr, page_type type) { + platform_assert(al->map_is_valid); debug_assert(rc_allocator_valid_extent_addr(al, addr)); uint64 extent_no = addr / al->cfg->io_cfg->extent_size; @@ -509,6 +536,7 @@ rc_allocator_alloc(rc_allocator *al, // IN uint64 *addr, // OUT page_type type) // IN { + platform_assert(al->map_is_valid); uint64 first_hand = al->hand % al->cfg->extent_capacity; uint64 hand; bool32 extent_is_free = FALSE; diff --git a/src/routing_filter.c b/src/routing_filter.c index 52938f54..e8a42978 100644 --- a/src/routing_filter.c +++ b/src/routing_filter.c @@ -1090,6 +1090,22 @@ routing_filter_inc_ref(cache *cc, routing_filter *filter) mini_inc_ref(cc, meta_head); } +/* + * Record the reference a holder has on this filter during a crash-recovery + * rebuild, enumerating its extents if this is the first one to reach it. The + * recovery counterpart of routing_filter_inc_ref(); see + * mini_recover_references(). An empty filter owns nothing, exactly as in + * routing_filter_inc_ref(). + */ +platform_status +routing_filter_recover_allocations(cache *cc, routing_filter *filter) +{ + if (filter->num_fingerprints == 0) { + return STATUS_OK; + } + return mini_recover_references(cc, filter->meta_head, PAGE_TYPE_FILTER); +} + /* *---------------------------------------------------------------------- * routing_filter_dec_ref diff --git a/src/routing_filter.h b/src/routing_filter.h index 79c3a4d0..01558b39 100644 --- a/src/routing_filter.h +++ b/src/routing_filter.h @@ -157,6 +157,14 @@ routing_filter_lookup_async(routing_filter_lookup_async_state *state); void routing_filter_dec_ref(cache *cc, routing_filter *filter); +/* + * Rebuild the allocator references this filter holds after a crash. The + * recovery counterpart of routing_filter_inc_ref(); see + * mini_recover_references(). + */ +platform_status +routing_filter_recover_allocations(cache *cc, routing_filter *filter); + void routing_filter_inc_ref(cache *cc, routing_filter *filter); diff --git a/src/shard_log.c b/src/shard_log.c index 3f6934d9..016091fb 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -14,22 +14,40 @@ #include "data_internal.h" #include "platform_sleep.h" #include "platform_hash.h" +#include "platform_random.h" #include "platform_typed_alloc.h" #include "platform_assert.h" #include "platform_threads.h" #include "platform_sort.h" #include "poison.h" -#define SHARD_WAIT 1 -#define SHARD_UNMAPPED UINT64_MAX - -static uint64 shard_log_magic_idx = 0; - static const page_type shard_log_page_type_table[NUM_BLOB_BATCHES + 1] = { PAGE_TYPE_LOG, [1 ... NUM_BLOB_BATCHES] = PAGE_TYPE_BLOB, }; +static platform_status +shard_log_graduate_through(shard_log *log, log_durable_ticket target); + +static platform_status +shard_log_wait_for_ticket_to_be_durable(shard_log *log, + log_durable_ticket target); + +static void +shard_log_destroy(shard_log *log); + +#if SPLINTER_DEBUG +static inline void +shard_log_run_test_hook(shard_log *log, + shard_log_test_hook_event event, + uint64 group_id) +{ + if (log->test_hook != NULL) { + log->test_hook(log->test_hook_arg, event, group_id); + } +} +#endif + static inline uint64 shard_log_page_size(shard_log_config *cfg) { @@ -56,12 +74,181 @@ shard_log_checksum(shard_log_config *cfg, page_handle *page) } static inline shard_log_thread_data * -shard_log_get_thread_data(shard_log *log, threadid thr_id) +shard_log_get_thread_data(shard_log_group *group, threadid thr_id) +{ + return &group->thread_data[thr_id]; +} + +static inline shard_log_reservation_slot * +shard_log_get_reservation_slot(shard_log *log, threadid tid) +{ + platform_assert(tid < MAX_THREADS); + return &log->reservation_slots[tid]; +} + +/* + * A scanner must not miss the sequentially consistent initial publication of + * a reservation which selected the old accepting group. Keep these scans SC; + * if they instead observe the later release-clear, that acquire publishes the + * protected staging writes and per-thread counters. + */ +static inline uint64 +shard_log_reservation_slot_load(shard_log *log, threadid tid) +{ + return __atomic_load_n(&shard_log_get_reservation_slot(log, tid)->ticket, + __ATOMIC_SEQ_CST); +} + +static inline uint64 +shard_log_reservation_slot_load_relaxed(shard_log *log, threadid tid) +{ + return __atomic_load_n(&shard_log_get_reservation_slot(log, tid)->ticket, + __ATOMIC_RELAXED); +} + +/* + * The initial hazard publication and the accepting-pointer load below form the + * store/load half of the reservation protocol. Keep the exchange sequentially + * consistent; as a bonus, its returned value performs the non-nesting check + * without a separate atomic load. + */ +static inline void +shard_log_reservation_slot_publish(shard_log *log, threadid tid, uint64 ticket) +{ + uint64 previous = + __atomic_exchange_n(&shard_log_get_reservation_slot(log, tid)->ticket, + ticket, + __ATOMIC_SEQ_CST); + platform_assert(previous == 0, + "log reservations may not be nested on one thread"); +} + +/* Replacing a conservative lower bound by the selected group's exact id. */ +static inline void +shard_log_reservation_slot_refine(shard_log *log, threadid tid, uint64 ticket) +{ + __atomic_store_n(&shard_log_get_reservation_slot(log, tid)->ticket, + ticket, + __ATOMIC_RELAXED); +} + +/* Publish all work protected by the reservation before withdrawing it. */ +static inline void +shard_log_reservation_slot_clear(shard_log *log, threadid tid) +{ + __atomic_store_n( + &shard_log_get_reservation_slot(log, tid)->ticket, 0, __ATOMIC_RELEASE); +} + +/* + * Publish a lower-bound hazard before loading the accepting pointer. The + * accepting pointer is installed before its ticket advances, so a reader that + * observes the newer ticket must also observe the newer pointer. A reader that + * observes the older ticket protects either pointer until it refines its slot + * to the selected group's exact ticket. + */ +static shard_log_group * +shard_log_publish_reservation(shard_log *log, threadid tid, uint64 *ticket_out) +{ + uint64 lower = __atomic_load_n(&log->accepting.id, __ATOMIC_SEQ_CST); + platform_assert(lower != 0); + shard_log_reservation_slot_publish(log, tid, lower); + + shard_log_group *group = + __atomic_load_n(&log->accepting.group, __ATOMIC_SEQ_CST); + if (group == NULL) { + shard_log_reservation_slot_clear(log, tid); + *ticket_out = 0; + return NULL; + } + + uint64 ticket = group->id; + platform_assert(ticket != 0); + platform_assert(ticket >= lower); + if (ticket != lower) { + shard_log_reservation_slot_refine(log, tid, ticket); + } + *ticket_out = ticket; + return group; +} + +static inline bool32 +shard_log_atomic_bool_load(const bool32 *value) +{ + return __atomic_load_n(value, __ATOMIC_SEQ_CST); +} + +/* Set-only values avoid a write or locked operation after their first set. */ +static inline void +shard_log_atomic_bool_set_once(bool32 *value) +{ + if (!shard_log_atomic_bool_load(value)) { + bool32 expected = FALSE; + (void)__atomic_compare_exchange_n( + value, &expected, TRUE, FALSE, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + } +} + +/* Begin calls are externally excluded from deinit, so the owner's reference + * guarantees that this relaxed increment cannot resurrect a dead handle. */ +static inline void +shard_log_handle_ref_acquire(shard_log *log) +{ + debug_assert(!__atomic_load_n(&log->owner_released, __ATOMIC_ACQUIRE)); + uint64 old = __atomic_fetch_add(&log->handle_refs, 1, __ATOMIC_RELAXED); + platform_assert(old != 0); +} + +/* Release either the owner reference or a durability-ticket reference. */ +static void +shard_log_handle_ref_release(shard_log *log) +{ + uint64 old = __atomic_fetch_sub(&log->handle_refs, 1, __ATOMIC_ACQ_REL); + platform_assert(old != 0, "log handle reference consumed more than once"); + if (old == 1) { + debug_assert(__atomic_load_n(&log->owner_released, __ATOMIC_ACQUIRE), + "durability ticket consumed more than once"); + shard_log_destroy(log); + } +} + +static inline log_durable_ticket +shard_log_graduated_ticket_load(const shard_log *log) { - return &log->thread_data[thr_id]; + return __atomic_load_n(&log->graduated_ticket, __ATOMIC_ACQUIRE); } -page_handle * +static inline void +shard_log_graduated_ticket_store(shard_log *log, log_durable_ticket ticket) +{ + __atomic_store_n(&log->graduated_ticket, ticket, __ATOMIC_RELEASE); +} + +static inline log_durable_ticket +shard_log_durable_ticket_load(const shard_log *log) +{ + return __atomic_load_n(&log->durable_ticket, __ATOMIC_ACQUIRE); +} + +static inline void +shard_log_durable_ticket_store(shard_log *log, log_durable_ticket ticket) +{ + __atomic_store_n(&log->durable_ticket, ticket, __ATOMIC_RELEASE); +} + +static bool32 +shard_log_operations_are_after(shard_log *log, log_durable_ticket ticket) +{ + for (threadid tid = 0; tid < MAX_THREADS; tid++) { + uint64 reserved = shard_log_reservation_slot_load(log, tid); + if (reserved != 0 && reserved <= ticket) { + return FALSE; + } + } + return TRUE; +} + +static page_handle * shard_log_alloc(shard_log *log, uint64 *next_extent) { uint64 addr = mini_alloc_page(&log->mini, 0, next_extent); @@ -148,50 +335,726 @@ log_entry_next(log_entry *le) return (log_entry *)((char *)le + sizeof_log_entry(le)); } +static platform_status +shard_log_measure_page_records(cache *cc, + shard_log_config *cfg, + page_handle *page, + uint64 first_needed_generation, + uint64 *num_entries_out, + uint64 *contents_size_out) +{ + *num_entries_out = 0; + *contents_size_out = 0; + for (log_entry *le = first_log_entry(page->data); + !terminal_log_entry(cfg, page->data, le); + le = log_entry_next(le)) + { + if (le->memtable_generation < first_needed_generation) { + continue; + } + if (log_entry_message_is_blob(le)) { + message msg = log_entry_message(cc, le); + platform_status rc = message_validate(msg); + if (!SUCCESS(rc)) { + return rc; + } + } + + uint64 entry_size = sizeof_log_entry(le); + if (*num_entries_out == UINT64_MAX + || entry_size > UINT64_MAX - *contents_size_out) + { + return STATUS_LIMIT_EXCEEDED; + } + (*num_entries_out)++; + *contents_size_out += entry_size; + } + return STATUS_OK; +} + +/* Reset a staging buffer to an empty page image. */ +static void +shard_log_reset_buffer(shard_log *log, shard_log_thread_data *thread_data) +{ + platform_assert(thread_data->incache_page == NULL); + shard_log_hdr *hdr = (shard_log_hdr *)thread_data->buf; + hdr->nonce = log->nonce; + hdr->num_entries = 0; + // next_extent_addr and checksum are only knowable once a page has been + // allocated for this image; see shard_log_graduate_buffer_internal(). + thread_data->offset = sizeof(shard_log_hdr); + thread_data->state = SHARD_LOG_BUFFER_OPEN; +} + +static inline uint64 +shard_log_buffer_payload_size(const shard_log_thread_data *thread_data) +{ + platform_assert(thread_data->offset >= sizeof(shard_log_hdr)); + return thread_data->offset - sizeof(shard_log_hdr); +} + +/* + * Move every staged record from src to dst. Both images remain private and + * mutable until graduation freezes them, so copying the raw payload preserves + * the already-packed record representation. The caller has checked capacity. + */ +static void +shard_log_merge_open_buffers(shard_log *log, + shard_log_thread_data *dst, + shard_log_thread_data *src) +{ + platform_assert(dst != src); + platform_assert(dst->state == SHARD_LOG_BUFFER_OPEN); + platform_assert(src->state == SHARD_LOG_BUFFER_OPEN); + + uint64 src_payload = shard_log_buffer_payload_size(src); + platform_assert(src_payload != 0); + platform_assert(src_payload <= shard_log_page_size(log->cfg) - dst->offset); + + shard_log_hdr *dst_hdr = (shard_log_hdr *)dst->buf; + shard_log_hdr *src_hdr = (shard_log_hdr *)src->buf; + platform_assert(src_hdr->num_entries != 0); + platform_assert(dst_hdr->num_entries <= UINT16_MAX - src_hdr->num_entries); + + memcpy( + dst->buf + dst->offset, src->buf + sizeof(shard_log_hdr), src_payload); + dst->offset += src_payload; + dst_hdr->num_entries += src_hdr->num_entries; + shard_log_reset_buffer(log, src); +} + +_Static_assert(IS_POWER_OF_2(MAX_THREADS), + "log packing tree requires a power-of-two thread count"); +_Static_assert(2 * MAX_THREADS < UINT16_MAX, + "log packing tree indices must fit in uint16"); + +#define SHARD_LOG_PACK_NO_BIN UINT16_MAX + +typedef struct shard_log_pack_sort_ctxt { + const uint16 *payloads; +} shard_log_pack_sort_ctxt; + static int -get_new_page_for_thread(shard_log *log, - shard_log_thread_data *thread_data, - page_handle **page) +shard_log_compare_buffer_size_desc(const void *lhs, const void *rhs, void *arg) { - uint64 next_extent; + const uint16 lhs_i = *(const uint16 *)lhs; + const uint16 rhs_i = *(const uint16 *)rhs; + shard_log_pack_sort_ctxt *ctxt = arg; - *page = shard_log_alloc(log, &next_extent); - if (*page == NULL) { - return -1; + uint16 lhs_size = ctxt->payloads[lhs_i]; + uint16 rhs_size = ctxt->payloads[rhs_i]; + + if (lhs_size != rhs_size) { + return lhs_size > rhs_size ? -1 : 1; + } + return (lhs_i > rhs_i) - (lhs_i < rhs_i); +} + +/* + * A complete max tree over the remaining capacities of the bins created so + * far. Leaves begin at MAX_THREADS; leaf/bin order is creation order. A + * left-first descent therefore finds the first bin into which an item fits. + */ +static void +shard_log_pack_tree_set(uint16 tree[2 * MAX_THREADS], + uint16 bin_i, + uint16 capacity) +{ + platform_assert(bin_i < MAX_THREADS); + uint16 node = MAX_THREADS + bin_i; + tree[node] = capacity; + while (node != 1) { + node /= 2; + uint16 left = 2 * node; + uint16 right = left + 1; + tree[node] = MAX(tree[left], tree[right]); + } +} + +static uint16 +shard_log_pack_tree_find_first(const uint16 tree[2 * MAX_THREADS], + uint16 required) +{ + platform_assert(required != 0); + if (tree[1] < required) { + return SHARD_LOG_PACK_NO_BIN; + } + + uint16 node = 1; + while (node < MAX_THREADS) { + uint16 left = 2 * node; + node = tree[left] >= required ? left : left + 1; + } + return node - MAX_THREADS; +} + +/* + * Compact the still-mutable buffers before closing a group. Thread 0 is never + * graduated as an ordinary page: it is kept for the final page carrying the + * group count. This makes a small group one page instead of data plus an empty + * terminator. First Fit Decreasing gives tight page packing without a + * quadratic placement scan: the max tree finds the first existing page with + * sufficient capacity in logarithmic time. + * + * A close retry may find another thread's image INCACHE from a failed + * writeback-set enrollment. Never inspect or alter such a frozen image. + * Successful graduations reset their buffers to empty OPEN images, which may + * safely be reused as destinations for records that have not yet been frozen. + * Since a source is reset in the same step that copies its payload, retries + * cannot duplicate records. + */ +static void +shard_log_pack_open_buffers(shard_log *log, shard_log_group *group) +{ + uint64 page_size = shard_log_page_size(log->cfg); + uint64 page_capacity = page_size - sizeof(shard_log_hdr); + platform_assert(page_capacity <= UINT16_MAX); + + shard_log_thread_data *final = shard_log_get_thread_data(group, 0); + platform_assert(final->state == SHARD_LOG_BUFFER_OPEN); + + uint16 items[MAX_THREADS]; + uint16 payloads[MAX_THREADS] = {0}; + uint16 num_items = 0; + uint64 total_payload = 0; + for (uint16 thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(group, thr_i); + if (thread_data->state == SHARD_LOG_BUFFER_OPEN) { + uint64 payload = shard_log_buffer_payload_size(thread_data); + if (payload != 0) { + platform_assert(payload <= page_capacity); + items[num_items++] = thr_i; + payloads[thr_i] = (uint16)payload; + total_payload += payload; + } + } + } + + /* Frequent barriers normally close a collection of small buffers. */ + if (total_payload <= page_capacity) { + for (uint16 item_i = 0; item_i < num_items; item_i++) { + shard_log_thread_data *src = + shard_log_get_thread_data(group, items[item_i]); + if (src != final) { + shard_log_merge_open_buffers(log, final, src); + } + } + return; + } + + if (num_items != 0) { + shard_log_pack_sort_ctxt sort_ctxt = {.payloads = payloads}; + uint16 temp; + platform_sort_slow(items, + num_items, + sizeof(items[0]), + shard_log_compare_buffer_size_desc, + &sort_ctxt, + &temp); + + uint16 bins[MAX_THREADS]; + uint16 tree[2 * MAX_THREADS] = {0}; + uint16 num_bins = 0; + + for (uint16 item_i = 0; item_i < num_items; item_i++) { + uint16 src_i = items[item_i]; + shard_log_thread_data *src = shard_log_get_thread_data(group, src_i); + uint16 src_payload = payloads[src_i]; + platform_assert(src->state == SHARD_LOG_BUFFER_OPEN); + platform_assert(src_payload != 0); + platform_assert(src_payload == shard_log_buffer_payload_size(src)); + + uint16 bin_i = shard_log_pack_tree_find_first(tree, src_payload); + if (bin_i == SHARD_LOG_PACK_NO_BIN) { + platform_assert(num_bins < MAX_THREADS); + bin_i = num_bins++; + bins[bin_i] = src_i; + shard_log_pack_tree_set( + tree, bin_i, (uint16)(page_capacity - src_payload)); + continue; + } + + shard_log_thread_data *dst = + shard_log_get_thread_data(group, bins[bin_i]); + shard_log_merge_open_buffers(log, dst, src); + shard_log_pack_tree_set( + tree, bin_i, (uint16)(page_size - dst->offset)); + } + + /* The final page must reside in thread 0 regardless of FFD's ordering. */ + if (shard_log_buffer_payload_size(final) == 0) { + platform_assert(num_bins != 0); + shard_log_thread_data *src = shard_log_get_thread_data(group, bins[0]); + platform_assert(src != final); + shard_log_merge_open_buffers(log, final, src); + } + } + + /* If any mutable payload remains, thread 0 is the final data page. */ + bool32 have_open_payload = FALSE; + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(group, thr_i); + have_open_payload |= thread_data->state == SHARD_LOG_BUFFER_OPEN + && shard_log_buffer_payload_size(thread_data) != 0; + } + platform_assert(!have_open_payload + || shard_log_buffer_payload_size(final) != 0); +} + +static void +shard_log_group_reset(shard_log *log, shard_log_group *group, uint64 id) +{ + group->id = id; + group->state = SHARD_LOG_GROUP_OPEN; + group->close = SHARD_LOG_CLOSE_NONE; + group->page_count = 0; + group->next = NULL; + group->pool_next = NULL; + __atomic_store_n(&group->ever_used, FALSE, __ATOMIC_RELAXED); + __atomic_store_n(&group->append_error.r, STATUS_OK.r, __ATOMIC_RELAXED); + + uint64 page_size = shard_log_page_size(log->cfg); + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(group, thr_i); + platform_assert(thread_data->incache_page == NULL); + thread_data->buf = group->thread_buffers + thr_i * page_size; + thread_data->page_count = 0; + writeback_set_reset(&thread_data->wbset); + shard_log_reset_buffer(log, thread_data); + } +} + +/* Preserve all-I/O-first pipelining by calling these group helpers in passes. + */ +static platform_status +shard_log_group_retry_writebacks(shard_log_group *group) +{ + platform_status result = STATUS_OK; + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(group, thr_i); + platform_status rc = writeback_set_retry_incomplete(&thread_data->wbset); + if (SUCCESS(result) && !SUCCESS(rc)) { + result = rc; + } + } + return result; +} + +static platform_status +shard_log_group_wait_for_writebacks(shard_log_group *group) +{ + platform_status result = STATUS_OK; + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(group, thr_i); + platform_status rc = writeback_set_wait(&thread_data->wbset); + if (SUCCESS(result) && !SUCCESS(rc)) { + result = rc; + } + } + return result; +} + +static void +shard_log_group_reset_writebacks(shard_log_group *group) +{ + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(group, thr_i); + writeback_set_reset(&thread_data->wbset); } - thread_data->addr = (*page)->disk_addr; - shard_log_hdr *hdr = (shard_log_hdr *)(*page)->data; - hdr->magic = log->magic; - hdr->next_extent_addr = next_extent; - hdr->num_entries = 0; - thread_data->offset = sizeof(shard_log_hdr); - return 0; } -int -shard_log_write(log_handle *logh, - key tuple_key, - message msg, - uint64 memtable_generation, - uint64 leaf_generation) +static uint64 +shard_log_group_page_count(const shard_log_group *group) +{ + uint64 pages = 0; + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + const shard_log_thread_data *thread_data = &group->thread_data[thr_i]; + platform_assert(thread_data->page_count <= SHARD_LOG_PAGES_IN_GROUP_MASK, + "thread %lu allocated too many pages in group %lu", + thr_i, + group->id); + platform_assert(thread_data->page_count + <= SHARD_LOG_PAGES_IN_GROUP_MASK - pages, + "group %lu is too large to terminate", + group->id); + pages += thread_data->page_count; + } + return pages; +} + +static void +shard_log_group_free(shard_log *log, shard_log_group *group) +{ + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(group, thr_i); + if (thread_data->incache_page != NULL) { + cache_unget(log->cc, thread_data->incache_page); + thread_data->incache_page = NULL; + } + writeback_set_deinit(&thread_data->wbset); + } + platform_free(log->heap_id, group->thread_buffers); + platform_free(log->heap_id, group->thread_data); + platform_free(log->heap_id, group); +} + +#define SHARD_LOG_MAX_RETAINED_WBSET_ENTRIES 16 + +/* Keep useful small vectors, but do not retain storage sized for a huge cut. */ +static void +shard_log_group_trim_writeback_storage(shard_log *log, shard_log_group *group) +{ + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(group, thr_i); + platform_assert(writeback_set_num_requests(&thread_data->wbset) == 0); + if (vector_capacity(&thread_data->wbset.entries) + > SHARD_LOG_MAX_RETAINED_WBSET_ENTRIES) + { + writeback_set_deinit(&thread_data->wbset); + writeback_set_init(&thread_data->wbset, log->cc, log->heap_id); + } + } +} + +static shard_log_group * +shard_log_group_malloc(shard_log *log, bool32 emergency) +{ + uint64 page_size = shard_log_page_size(log->cfg); + + shard_log_group *group = TYPED_MALLOC(log->heap_id, group); + if (group == NULL) { + return NULL; + } + ZERO_CONTENTS(group); + + group->thread_data = + TYPED_ARRAY_MALLOC(log->heap_id, group->thread_data, MAX_THREADS); + if (group->thread_data == NULL) { + goto cleanup; + } + + group->thread_buffers = TYPED_ARRAY_MALLOC( + log->heap_id, group->thread_buffers, MAX_THREADS * page_size); + if (group->thread_buffers == NULL) { + goto cleanup; + } + + memset(group->thread_buffers, 0, MAX_THREADS * page_size); + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(group, thr_i); + thread_data->buf = group->thread_buffers + thr_i * page_size; + thread_data->incache_page = NULL; + writeback_set_init(&thread_data->wbset, log->cc, log->heap_id); + } + shard_log_group_reset(log, group, 0); + group->emergency = emergency; + return group; + +cleanup: + if (group->thread_buffers) { + platform_free(log->heap_id, group->thread_buffers); + } + if (group->thread_data) { + platform_free(log->heap_id, group->thread_data); + } + platform_free(log->heap_id, group); + return NULL; +} + +/* Return an unused candidate group to its allocation source. */ +static void +shard_log_put_unused_group(shard_log *log, shard_log_group *group) +{ + if (group == NULL) { + return; + } + shard_log_group_trim_writeback_storage(log, group); + if (group->emergency) { + platform_mutex_lock(&log->group_pool_lock); + group->pool_next = log->emergency_pool; + log->emergency_pool = group; + platform_mutex_unlock(&log->group_pool_lock); + return; + } + + platform_mutex_lock(&log->group_pool_lock); + if (log->reusable_pool_count < SHARD_LOG_NUM_REUSABLE_GROUPS) { + group->pool_next = log->reusable_pool; + log->reusable_pool = group; + log->reusable_pool_count++; + group = NULL; + } + platform_mutex_unlock(&log->group_pool_lock); + + if (group != NULL) { + shard_log_group_free(log, group); + } +} + +/* + * Reuse a recently retired ordinary group, then allocate normally. The two + * preallocated emergency groups are consumed only when the heap cannot provide + * a group, so normal bursts scale with device latency while low-memory progress + * retains a bounded reserve. + */ +static platform_status +shard_log_allocate_group(shard_log *log, shard_log_group **group_out) +{ + platform_mutex_lock(&log->group_pool_lock); + shard_log_group *group = log->reusable_pool; + if (group != NULL) { + log->reusable_pool = group->pool_next; + group->pool_next = NULL; + platform_assert(log->reusable_pool_count != 0); + log->reusable_pool_count--; + } + platform_mutex_unlock(&log->group_pool_lock); + + if (group != NULL) { + shard_log_group_reset(log, group, 0); + *group_out = group; + return STATUS_OK; + } + + group = shard_log_group_malloc(log, FALSE); + if (group != NULL) { + *group_out = group; + return STATUS_OK; + } + + platform_mutex_lock(&log->group_pool_lock); + group = log->emergency_pool; + if (group != NULL) { + log->emergency_pool = group->pool_next; + group->pool_next = NULL; + } + platform_mutex_unlock(&log->group_pool_lock); + if (group == NULL) { + *group_out = NULL; + return STATUS_NO_MEMORY; + } + shard_log_group_reset(log, group, 0); + *group_out = group; + return STATUS_OK; +} + +/* + * Turn a thread's staged image into an on-disk log page: allocate the page, + * copy the image in, and hand the write to the cache. A no-op if nothing has + * been staged. + * + * This is the only place a log page is written, and it writes each page exactly + * once, in full -- so there is never a partially-filled log page on disk to be + * rewritten later. + */ +static platform_status +shard_log_graduate_buffer_internal(shard_log *log, + shard_log_group *group, + shard_log_thread_data *thread_data, + bool32 final) +{ + shard_log_close_mode close = final ? group->close : SHARD_LOG_CLOSE_NONE; + uint64 page_size = shard_log_page_size(log->cfg); + bool32 close_group = final; + + platform_assert(!final || close != SHARD_LOG_CLOSE_NONE); + + debug_assert(thread_data->offset >= sizeof(shard_log_hdr)); + debug_assert(thread_data->offset <= page_size); + + if (thread_data->state == SHARD_LOG_BUFFER_OPEN) { + /* + * An empty buffer normally has nothing to contribute. Group termination + * deliberately uses an otherwise empty page as an explicit commit marker. + */ + if (thread_data->offset == sizeof(shard_log_hdr) && !close_group) { + return STATUS_OK; + } + + uint64 next_extent; + page_handle *page = shard_log_alloc(log, &next_extent); + if (page == NULL) { + platform_error_log("shard_log_graduate_buffer: out of log space\n"); + return STATUS_NO_SPACE; + } + + uint64 free_space = page_size - thread_data->offset; + if (sizeof(log_entry) <= free_space) { + log_entry_set_terminal( + (log_entry *)(thread_data->buf + thread_data->offset)); + } + + shard_log_hdr *staged = (shard_log_hdr *)thread_data->buf; + staged->next_extent_addr = next_extent; + staged->group_id = group->id; + /* + * Count exactly once, when the image obtains its permanent cache page. If + * writeback enrollment fails, INCACHE retains both the page and this + * ordinal for a retry. + */ + thread_data->page_count++; + if (close_group) { + uint64 pages = shard_log_group_page_count(group); + platform_assert(pages <= SHARD_LOG_PAGES_IN_GROUP_MASK, + "group %lu is too large to terminate: %lu pages", + group->id, + pages); + group->page_count = pages; + staged->pages_in_group = + (uint32)pages + | (close == SHARD_LOG_CLOSE_STREAM ? SHARD_LOG_END_OF_STREAM : 0); + } else { + staged->pages_in_group = 0; + } + + memcpy(page->data, thread_data->buf, page_size); + ((shard_log_hdr *)page->data)->checksum = + shard_log_checksum(log->cfg, page); + + cache_unlock(log->cc, page); + cache_unclaim(log->cc, page); + thread_data->incache_page = page; // retain our cache reference for retry + thread_data->state = SHARD_LOG_BUFFER_INCACHE; + } + + platform_assert(thread_data->state == SHARD_LOG_BUFFER_INCACHE); + platform_status rc = writeback_set_add_page( + &thread_data->wbset, thread_data->incache_page, PAGE_TYPE_LOG); + if (!SUCCESS(rc)) { + return rc; + } + + cache_unget(log->cc, thread_data->incache_page); + thread_data->incache_page = NULL; + shard_log_reset_buffer(log, thread_data); + return STATUS_OK; +} + +static platform_status +shard_log_graduate_ordinary_buffer(shard_log *log, + shard_log_group *group, + shard_log_thread_data *thread_data) +{ + return shard_log_graduate_buffer_internal(log, group, thread_data, FALSE); +} + +static platform_status +shard_log_graduate_final_buffer(shard_log *log, shard_log_group *group) +{ + platform_assert(group->state == SHARD_LOG_GROUP_TERMINATING); + return shard_log_graduate_buffer_internal( + log, group, shard_log_get_thread_data(group, 0), TRUE); +} + +static void +shard_log_write_reserve(log_handle *logh, log_write_token *token) +{ + shard_log *log = (shard_log *)logh; + threadid tid = platform_get_tid(); + + uint64 ticket; + shard_log_group *group = shard_log_publish_reservation(log, tid, &ticket); + platform_assert(group != NULL, + "log_write_reserve called after the stream was sealed"); + + shard_log_atomic_bool_set_once(&group->ever_used); + token->log = logh; + token->internal = group; + token->owner_tid = tid; + token->internal_ticket = ticket; +} + +static platform_status +shard_log_end_append(shard_log *log, + shard_log_group *group, + threadid tid, + uint64 ticket, + bool32 accepted_record, + platform_status append_rc) +{ + platform_assert(shard_log_reservation_slot_load_relaxed(log, tid) == ticket); + if (!SUCCESS(append_rc)) { + /* + * The caller's update was already visible before it entered the log. + * Remember the first missing append permanently; a later retry or error + * must not obscure the reason this recovery suffix is unusable. + */ + internal_platform_status expected = STATUS_OK.r; + (void)__atomic_compare_exchange_n(&group->append_error.r, + &expected, + append_rc.r, + FALSE, + __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST); + } + if (accepted_record) { + shard_log_atomic_bool_set_once(&log->has_records); + } + platform_status result = { + .r = __atomic_load_n(&group->append_error.r, __ATOMIC_SEQ_CST), + }; + + /* Publish every staging-buffer write before allowing group graduation. */ + shard_log_reservation_slot_clear(log, tid); + return result; +} + +static int +shard_log_write_reserved(log_write_token *token, + key tuple_key, + message msg, + uint64 memtable_generation, + uint64 leaf_generation) { debug_assert(key_is_user_key(tuple_key)); debug_assert(memtable_generation != INVALID_LOG_GENERATION); debug_assert(leaf_generation != INVALID_LOG_GENERATION); - shard_log *log = (shard_log *)logh; - cache *cc = log->cc; - merge_accumulator log_blob; - bool32 log_blob_inited = FALSE; - - uint64 max_entry_size = - shard_log_page_size(log->cfg) - sizeof(shard_log_hdr); - if (message_is_blob(msg) - || max_entry_size < log_entry_required_capacity(tuple_key, msg)) - { + platform_assert(token != NULL); + platform_assert(token->log != NULL); + platform_assert(token->internal != NULL); + + shard_log *log = (shard_log *)token->log; + shard_log_group *group = token->internal; + threadid tid = platform_get_tid(); + uint64 ticket = token->internal_ticket; + + platform_assert(token->owner_tid == tid, + "log reservations must be consumed by their owner thread"); + platform_assert(ticket == group->id); + platform_assert(shard_log_reservation_slot_load_relaxed(log, tid) == ticket, + "log write receipt does not match the active reservation"); + + /* Ownership is consumed on every return path from this point onward. */ + token->log = NULL; + token->internal = NULL; + token->owner_tid = INVALID_TID; + token->internal_ticket = 0; + + cache *cc = log->cc; + platform_status rc = STATUS_OK; + merge_accumulator log_blob; + bool32 log_blob_inited = FALSE; + bool32 accepted_record = FALSE; + shard_log_thread_data *thread_data = shard_log_get_thread_data(group, tid); + + uint64 page_size = shard_log_page_size(log->cfg); + uint64 max_entry_size = page_size - sizeof(shard_log_hdr); + bool32 input_is_blob = message_is_blob(msg); + uint64 new_entry_size = + input_is_blob ? 0 : log_entry_required_capacity(tuple_key, msg); + if (input_is_blob || max_entry_size < new_entry_size) { merge_accumulator_init(&log_blob, platform_get_heap_id()); - platform_status rc; - if (message_is_blob(msg)) { + if (input_is_blob) { rc = message_clone(&log->cfg->blob_cfg, cc, &log->mini, msg, &log_blob); } else { @@ -200,203 +1063,908 @@ shard_log_write(log_handle *logh, } if (!SUCCESS(rc)) { merge_accumulator_deinit(&log_blob); - return rc.r; + goto out; } msg = merge_accumulator_to_message(&log_blob); log_blob_inited = TRUE; + new_entry_size = log_entry_required_capacity(tuple_key, msg); + } + + if (log_blob_inited) { + /* + * Enroll the value before publishing the record into a staging buffer. + * A partial blob-writeback enrollment may leave harmless extra receipts, + * but it can no longer leave behind a record whose value was not covered. + */ + rc = blob_writeback(cc, message_slice(msg), &thread_data->wbset); + if (!SUCCESS(rc)) { + goto out; + } } - shard_log_thread_data *thread_data = - shard_log_get_thread_data(log, platform_get_tid()); + debug_assert(new_entry_size <= page_size - sizeof(shard_log_hdr)); - page_handle *page; - if (thread_data->addr == SHARD_UNMAPPED) { - if (get_new_page_for_thread(log, thread_data, &page)) { - if (log_blob_inited) { - merge_accumulator_deinit(&log_blob); + // Full, or retrying an earlier hand-over: finish that frozen image first. + if (thread_data->state != SHARD_LOG_BUFFER_OPEN + || page_size - thread_data->offset < new_entry_size) + { + /* + * Log pages must remain physically grouped. A later group may stage + * records immediately, but before it allocates its first page it helps + * graduate every predecessor (including each predecessor terminator). + */ + platform_assert(group->id >= SHARD_LOG_FIRST_GROUP_ID); + log_durable_ticket predecessor = group->id - 1; + rc = shard_log_graduate_through(log, predecessor); + if (SUCCESS(rc)) { + rc = shard_log_graduate_ordinary_buffer(log, group, thread_data); + } + if (!SUCCESS(rc)) { + goto out; + } + } + + log_entry *cursor = (log_entry *)(thread_data->buf + thread_data->offset); + cursor->memtable_generation = memtable_generation; + cursor->leaf_generation = leaf_generation; + copy_tuple_to_ondisk_tuple(&cursor->tuple, tuple_key, msg); + + ((shard_log_hdr *)thread_data->buf)->num_entries++; + + thread_data->offset += new_entry_size; + accepted_record = TRUE; + debug_assert(thread_data->offset <= page_size); + +out: + if (log_blob_inited) { + merge_accumulator_deinit(&log_blob); + } + rc = shard_log_end_append(log, group, tid, ticket, accepted_record, rc); + return rc.r; +} + +/* Caller holds graduate_lock or durability_lock, excluding reclamation. */ +static shard_log_group * +shard_log_find_group(shard_log *log, log_durable_ticket ticket) +{ + platform_assert(ticket != 0); + for (shard_log_group *group = log->groups_head; group != NULL; + group = group->next) + { + if (group->id == ticket) { + return group; + } + } + return NULL; +} + +/* graduate_lock is held; the returned cursor is valid until it is released. */ +static shard_log_group * +shard_log_find_next_group_to_graduate(shard_log *log, + log_durable_ticket next_ticket) +{ + shard_log_group *group = shard_log_find_group(log, next_ticket); + platform_assert(group != NULL); + platform_assert(group->state != SHARD_LOG_GROUP_OPEN); + return group; +} + +/* Wait for every operation which selected this group before its cut. */ +static void +shard_log_wait_for_operations(shard_log *log, log_durable_ticket ticket) +{ + uint64 wait = 100; + while (!shard_log_operations_are_after(log, ticket)) { + platform_sleep_ns(wait); + wait = wait > 2048 ? wait : 2 * wait; + } +} + +/* + * A cutter may select a successor after its pointer is published but before + * the preceding cutter advances accepting.id. After winning the next claim, + * wait for that final id store before modifying the selected group. This makes + * accepting.id the ordered publication handoff between adjacent cutters. + */ +static void +shard_log_wait_for_accepting_publication(shard_log *log, + shard_log_group *current) +{ + uint64 wait = 100; + debug_code(bool32 hook_called = FALSE); + while (TRUE) { + uint64 accepting_id = + __atomic_load_n(&log->accepting.id, __ATOMIC_ACQUIRE); + platform_assert(accepting_id <= current->id); + if (accepting_id == current->id) { + platform_assert( + __atomic_load_n(&log->accepting.group, __ATOMIC_ACQUIRE) + == current); + return; + } + + debug_code(if (!hook_called) { + shard_log_run_test_hook( + log, SHARD_LOG_TEST_ACCEPTING_HANDOFF_WAIT, current->id); + hook_called = TRUE; + }); + platform_sleep_ns(wait); + wait = wait > 2048 ? wait : 2 * wait; + } +} + +/* + * A make_durable_begin() caller which loses the installation claim may return + * before the winner has published the cut. The accepting id advances only + * after a normal successor is fully linked, while NULL is the final seal + * publication. Wait for either publication without contending with cutters. + */ +static void +shard_log_wait_for_cut(shard_log *log, log_durable_ticket ticket) +{ + uint64 wait = 100; + while (TRUE) { + uint64 accepting_id = + __atomic_load_n(&log->accepting.id, __ATOMIC_ACQUIRE); + if (ticket < accepting_id) { + return; + } + + shard_log_group *accepting = + __atomic_load_n(&log->accepting.group, __ATOMIC_ACQUIRE); + if (accepting == NULL) { + uint64 install_state = + __atomic_load_n(&log->install.state, __ATOMIC_ACQUIRE); + platform_assert(install_state & SHARD_LOG_INSTALL_TERMINAL_BIT); + platform_assert(ticket <= (install_state & SHARD_LOG_INSTALL_ID_MASK)); + return; + } + + platform_sleep_ns(wait); + wait = wait > 2048 ? wait : 2 * wait; + } +} + +/* graduate_lock must be held. */ +static platform_status +shard_log_graduate_group(shard_log *log, shard_log_group *group) +{ + platform_assert(group->close != SHARD_LOG_CLOSE_NONE); + platform_assert(shard_log_operations_are_after(log, group->id)); + platform_status append_error = { + .r = __atomic_load_n(&group->append_error.r, __ATOMIC_SEQ_CST), + }; + + /* + * A terminator is the on-disk commit record for the whole group. Emitting + * one after any append failed would make recovery skip a visible update and + * then accept the suffix as complete. Data pages already graduated before + * the failure remain harmless because recovery discards an unterminated + * group and everything after it. + */ + if (!SUCCESS(append_error)) { + return append_error; + } + + if (group->state == SHARD_LOG_GROUP_CLOSING) { + /* + * Pack only mutable images, then freeze every ordinary page before the + * reserved final page. A retry finishes INCACHE images without repacking + * them; thread 0 remains OPEN throughout this phase. + */ + shard_log_pack_open_buffers(log, group); + for (threadid thr_i = 1; thr_i < MAX_THREADS; thr_i++) { + platform_status rc = shard_log_graduate_ordinary_buffer( + log, group, shard_log_get_thread_data(group, thr_i)); + if (!SUCCESS(rc)) { + platform_error_log("shard_log_graduate_group: failed to flush " + "thread %lu in group %lu: %s\n", + thr_i, + group->id, + platform_status_to_string(rc)); + return rc; } - return -1; } - } else { - page = cache_get(cc, thread_data->addr, TRUE, PAGE_TYPE_LOG); - uint64 wait = 1; - while (!cache_try_claim(cc, page)) { - cache_unget(cc, page); - platform_sleep_ns(wait); - wait = wait > 1024 ? wait : 2 * wait; - page = cache_get(cc, thread_data->addr, TRUE, PAGE_TYPE_LOG); + group->state = SHARD_LOG_GROUP_TERMINATING; + } + + if (group->state == SHARD_LOG_GROUP_TERMINATING) { + platform_status rc = shard_log_graduate_final_buffer(log, group); + if (!SUCCESS(rc)) { + platform_error_log("shard_log_graduate_group: failed to terminate " + "group %lu: %s\n", + group->id, + platform_status_to_string(rc)); + return rc; } - cache_lock(cc, page); + group->state = SHARD_LOG_GROUP_DURABILITY_PENDING; } - shard_log_hdr *hdr = (shard_log_hdr *)page->data; - log_entry *cursor = (log_entry *)(page->data + thread_data->offset); - uint64 new_entry_size = log_entry_required_capacity(tuple_key, msg); - uint64 free_space = shard_log_page_size(log->cfg) - thread_data->offset; - debug_assert(new_entry_size - <= shard_log_page_size(log->cfg) - sizeof(shard_log_hdr)); + platform_assert(group->state == SHARD_LOG_GROUP_DURABILITY_PENDING); + return STATUS_OK; +} - if (free_space < new_entry_size) { - if (sizeof(log_entry) <= free_space) { - log_entry_set_terminal(cursor); +/* + * Allocate log pages in group-id order. Later groups may already contain + * private staged records, but no page of N+1 is allocated until N's terminator + * has obtained its permanent address and writeback receipt. + */ +static platform_status +shard_log_graduate_through(shard_log *log, log_durable_ticket target) +{ + /* The release publication makes an already-graduated target lock-free. */ + if (target <= shard_log_graduated_ticket_load(log)) { + return STATUS_OK; + } + + shard_log_wait_for_cut(log, target); + + /* Another waiter may have graduated the target while its cut was pending. */ + if (target <= shard_log_graduated_ticket_load(log)) { + return STATUS_OK; + } + + platform_status rc = platform_mutex_lock(&log->graduate_lock); + if (!SUCCESS(rc)) { + return rc; + } + + log_durable_ticket graduated = shard_log_graduated_ticket_load(log); + shard_log_group *group = NULL; + if (graduated < target) { + group = shard_log_find_next_group_to_graduate(log, graduated + 1); + } + + while (graduated < target) { + log_durable_ticket next_ticket = graduated + 1; + platform_assert(group != NULL); + platform_assert(group->id == next_ticket); + + if (!shard_log_operations_are_after(log, next_ticket)) { + /* + * Do not hold graduate_lock while draining the closed group. One of + * its reserved writers may be blocked in this function helping finish + * its predecessor before it can graduate a full buffer and consume + * its reservation. Closed groups cannot acquire new reservations, so + * it is safe to drop the driver lock, wait, and re-evaluate the + * frontier. + */ + platform_mutex_unlock(&log->graduate_lock); + shard_log_wait_for_operations(log, next_ticket); + rc = platform_mutex_lock(&log->graduate_lock); + if (!SUCCESS(rc)) { + return rc; + } + graduated = shard_log_graduated_ticket_load(log); + group = graduated < target + ? shard_log_find_next_group_to_graduate(log, graduated + 1) + : NULL; + continue; + } + + rc = shard_log_graduate_group(log, group); + if (!SUCCESS(rc)) { + break; + } + + platform_assert(shard_log_graduated_ticket_load(log) == graduated); + shard_log_graduated_ticket_store(log, next_ticket); + + graduated = next_ticket; + group = group->next; + } + + platform_status unlock_rc = platform_mutex_unlock(&log->graduate_lock); + if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { + rc = unlock_rc; + } + return rc; +} + +/* + * Publish and remove exactly the newly durable numeric prefix. + * durability_lock is held and the device barrier has succeeded. + */ +static void +shard_log_publish_durable_prefix(shard_log *log, + log_durable_ticket previous_durable, + log_durable_ticket target) +{ + platform_assert(previous_durable < target); + platform_assert(previous_durable == shard_log_durable_ticket_load(log)); + + platform_mutex_lock(&log->graduate_lock); + + shard_log_group *reclaim_head = log->groups_head; + shard_log_group *remaining = reclaim_head; + for (log_durable_ticket ticket = previous_durable + 1; ticket <= target; + ticket++) + { + platform_assert(remaining != NULL); + platform_assert(remaining->id == ticket); + platform_assert(remaining->state == SHARD_LOG_GROUP_DURABILITY_PENDING); + shard_log_group_reset_writebacks(remaining); + remaining = remaining->next; + } + log->groups_head = remaining; + shard_log_durable_ticket_store(log, target); + platform_mutex_unlock(&log->graduate_lock); + + /* The release publication above lets coalesced waiters return while this + * caller performs allocation housekeeping on the now-detached objects. */ + while (reclaim_head != remaining) { + shard_log_group *group = reclaim_head; + reclaim_head = group->next; + group->next = NULL; + shard_log_put_unused_group(log, group); + } +} + +/* Wait/writeback/barrier half shared by ticket wait and seal. */ +static platform_status +shard_log_wait_for_ticket_to_be_durable(shard_log *log, + log_durable_ticket target) +{ + /* Release publication makes an already-durable target lock-free. */ + if (target <= shard_log_durable_ticket_load(log)) { + return STATUS_OK; + } + + platform_status rc = shard_log_graduate_through(log, target); + if (!SUCCESS(rc)) { + return rc; + } + + /* Another waiter may have completed the target while we graduated it. */ + if (target <= shard_log_durable_ticket_load(log)) { + return STATUS_OK; + } + + rc = platform_mutex_lock(&log->durability_lock); + if (!SUCCESS(rc)) { + return rc; + } + + platform_status result = STATUS_OK; + log_durable_ticket durable = shard_log_durable_ticket_load(log); + if (durable < target) { + /* + * durability_lock excludes reclamation, so this cursor and the immutable + * closed-group links through target remain valid across all three passes. + */ + shard_log_group *first = shard_log_find_group(log, durable + 1); + platform_assert(first != NULL); + platform_assert(first->state == SHARD_LOG_GROUP_DURABILITY_PENDING); + + /* + * Issue every needed retry before waiting for any group, preserving the + * same all-I/O-first pipelining as writeback_set itself. + */ + shard_log_group *group = first; + for (log_durable_ticket ticket = durable + 1; ticket <= target; ticket++) + { + platform_assert(group != NULL); + platform_assert(group->id == ticket); + platform_assert(group->state == SHARD_LOG_GROUP_DURABILITY_PENDING); + + platform_status retry_rc = shard_log_group_retry_writebacks(group); + if (SUCCESS(result) && !SUCCESS(retry_rc)) { + result = retry_rc; + } + group = group->next; } - hdr->checksum = shard_log_checksum(log->cfg, page); - cache_unlock(cc, page); - cache_unclaim(cc, page); - cache_page_writeback(cc, page, FALSE, PAGE_TYPE_LOG); - cache_unget(cc, page); + group = first; + for (log_durable_ticket ticket = durable + 1; ticket <= target; ticket++) + { + platform_assert(group != NULL); + platform_assert(group->id == ticket); - if (get_new_page_for_thread(log, thread_data, &page)) { - if (log_blob_inited) { - merge_accumulator_deinit(&log_blob); + platform_status wait_rc = shard_log_group_wait_for_writebacks(group); + if (SUCCESS(result) && !SUCCESS(wait_rc)) { + result = wait_rc; } - return -1; + group = group->next; + } + + if (SUCCESS(result)) { + result = cache_durable_barrier(log->cc); + } + + if (SUCCESS(result)) { + shard_log_publish_durable_prefix(log, durable, target); } - cursor = (log_entry *)(page->data + thread_data->offset); - hdr = (shard_log_hdr *)page->data; } - cursor->memtable_generation = memtable_generation; - cursor->leaf_generation = leaf_generation; - copy_tuple_to_ondisk_tuple(&cursor->tuple, tuple_key, msg); + platform_status unlock_rc = platform_mutex_unlock(&log->durability_lock); + if (SUCCESS(result) && !SUCCESS(unlock_rc)) { + result = unlock_rc; + } + return result; +} + +/* Install a fresh accepting group and return the closed group's ticket. */ +static platform_status +shard_log_make_durable_begin(log_handle *logh, log_durable_ticket *ticket_out) +{ + platform_assert(ticket_out != NULL); + *ticket_out = 0; + shard_log *log = (shard_log *)logh; + threadid tid = platform_get_tid(); + + log_durable_ticket target; + shard_log_group *current = shard_log_publish_reservation(log, tid, &target); + if (current == NULL) { + uint64 install_state = + __atomic_load_n(&log->install.state, __ATOMIC_SEQ_CST); + platform_assert(install_state & SHARD_LOG_INSTALL_TERMINAL_BIT); + target = install_state & SHARD_LOG_INSTALL_ID_MASK; + platform_assert(target >= SHARD_LOG_FIRST_GROUP_ID); + shard_log_handle_ref_acquire(log); + + *ticket_out = target; + return STATUS_OK; + } - hdr->num_entries++; + if (!shard_log_atomic_bool_load(¤t->ever_used)) { + platform_assert(target >= SHARD_LOG_FIRST_GROUP_ID); + target--; + shard_log_handle_ref_acquire(log); - thread_data->offset += new_entry_size; - debug_assert(thread_data->offset <= shard_log_page_size(log->cfg)); + shard_log_reservation_slot_clear(log, tid); + *ticket_out = target; + return STATUS_OK; + } - cache_unlock(cc, page); - cache_unclaim(cc, page); - cache_unget(cc, page); + platform_assert(target < SHARD_LOG_INSTALL_ID_MASK); + shard_log_reservation_slot_clear(log, tid); - if (log_blob_inited) { - platform_status rc = blob_sync(cc, message_slice(msg)); - merge_accumulator_deinit(&log_blob); + /* + * Avoid allocating a candidate when another caller has already claimed this + * cut. The claimant's remaining path is infallible, so this ticket can be + * returned before that caller finishes publishing it. + */ + uint64 install_state = + __atomic_load_n(&log->install.state, __ATOMIC_SEQ_CST); + shard_log_group *candidate = NULL; + if (install_state == target) { + /* Allocate the group before claiming the installation task, primarily to + * avoid having to deal with failures after claiming the job. */ + platform_status rc = shard_log_allocate_group(log, &candidate); if (!SUCCESS(rc)) { - return rc.r; + /* + * Allocation raced with another caller taking this cut. Once the + * install state advances, that caller's publication path is + * infallible, so our original target remains a successful cut even + * though we could not allocate its successor ourselves. + */ + install_state = __atomic_load_n(&log->install.state, __ATOMIC_SEQ_CST); + if (install_state == target) { + return rc; + } + } else { + uint64 expected_state = target; + if (__atomic_compare_exchange_n(&log->install.state, + &expected_state, + target + 1, + FALSE, + __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST)) + { + /* + * Although we've already released our reservation, a successful + * claim proves that target was never cut and current remains valid. + * The claim makes the remaining installation path infallible. A + * preceding cutter may still be finishing the pointer-first + * publication by which we selected current, so wait for its id + * handoff before changing current or publishing our successor. + */ + shard_log_wait_for_accepting_publication(log, current); + platform_assert( + __atomic_load_n(&log->install.state, __ATOMIC_SEQ_CST) + == target + 1); + debug_assert( + __atomic_load_n(&log->accepting.group, __ATOMIC_SEQ_CST) + == current); + platform_assert(current->id == target); + platform_assert(current->state == SHARD_LOG_GROUP_OPEN); + + current->state = SHARD_LOG_GROUP_CLOSING; + current->close = SHARD_LOG_CLOSE_GROUP; + + candidate->id = target + 1; + current->next = candidate; + + shard_log_handle_ref_acquire(log); + + /* Pointer first, ticket last: see shard_log_publish_reservation(). + */ + __atomic_store_n( + &log->accepting.group, candidate, __ATOMIC_SEQ_CST); + debug_code(shard_log_run_test_hook( + log, SHARD_LOG_TEST_SUCCESSOR_POINTER_PUBLISHED, candidate->id)); + __atomic_store_n( + &log->accepting.id, candidate->id, __ATOMIC_SEQ_CST); + + *ticket_out = target; + return STATUS_OK; + } + install_state = expected_state; } } - return 0; + /* Another installer or seal owns this target; never chase its successor. */ + platform_assert((install_state & SHARD_LOG_INSTALL_ID_MASK) >= target); + shard_log_handle_ref_acquire(log); + + shard_log_put_unused_group(log, candidate); + *ticket_out = target; + return STATUS_OK; +} + +static platform_status +shard_log_make_durable_wait(log_handle *logh, log_durable_ticket ticket) +{ + shard_log *log = (shard_log *)logh; + platform_status rc = shard_log_wait_for_ticket_to_be_durable(log, ticket); + shard_log_handle_ref_release(log); + return rc; } -/* - * shard_log_seal -- - * - * Finalize and retire a log stream, terminally. Finalizes every currently - * active per-thread append page (bounded by MAX_THREADS; it does not walk - * the historical log): a terminal record (where there is room) and checksum - * make each page readable by shard_log_iterator_init(). Then it releases - * the mini-allocator's unused reserve and frees the handle. After seal the - * handle is invalid; the caller retains the identity it captured earlier - * (shard_log_get_head(), fixed at creation) to reopen the stream - * for replay and, eventually, to free its extents via log_dec_ref(). - * - * The caller must prevent concurrent shard_log_write() and seal calls. - * seal itself issues no writeback or durable barrier: to make the sealed - * pages durable, the caller takes cache_writeback_dirty() followed by a - * durable barrier. - */ platform_status shard_log_seal(log_handle *logh) { - shard_log *log = (shard_log *)logh; - cache *cc = log->cc; - - for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { - shard_log_thread_data *thread_data = - shard_log_get_thread_data(log, thr_i); - uint64 addr = thread_data->addr; - if (addr == SHARD_UNMAPPED) { - continue; + shard_log *log = (shard_log *)logh; + uint64 wait = 100; + log_durable_ticket target; + threadid tid = platform_get_tid(); + + platform_assert( + shard_log_reservation_slot_load_relaxed(log, tid) == 0, + "log_seal cannot wait on its calling thread's write reservation"); + + while (TRUE) { + shard_log_group *current = + shard_log_publish_reservation(log, tid, &target); + if (current == NULL) { + uint64 install_state = + __atomic_load_n(&log->install.state, __ATOMIC_SEQ_CST); + platform_assert(install_state & SHARD_LOG_INSTALL_TERMINAL_BIT); + target = install_state & SHARD_LOG_INSTALL_ID_MASK; + platform_assert(target >= SHARD_LOG_FIRST_GROUP_ID); + break; } - page_handle *page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); - uint64 wait = 1; - while (!cache_try_claim(cc, page)) { - /* - * Even though the stream is quiescent (no concurrent writers/seals), - * the background cache evictor can transiently hold the claim on a - * cleaned log page before it drains our read-ref, so we still retry - * rather than assert. - */ - cache_unget(cc, page); + platform_assert(target == current->id); + uint64 expected_state = target; + uint64 terminal_state = SHARD_LOG_INSTALL_TERMINAL_BIT | target; + if (!__atomic_compare_exchange_n(&log->install.state, + &expected_state, + terminal_state, + FALSE, + __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST)) + { + shard_log_reservation_slot_clear(log, tid); + if (expected_state & SHARD_LOG_INSTALL_TERMINAL_BIT) { + target = expected_state & SHARD_LOG_INSTALL_ID_MASK; + platform_assert(target >= SHARD_LOG_FIRST_GROUP_ID); + while (__atomic_load_n(&log->accepting.group, __ATOMIC_ACQUIRE) + != NULL) + { + platform_sleep_ns(wait); + wait = wait > 2048 ? wait : 2 * wait; + } + break; + } + + /* An installer owns this generation; let it publish and retry. */ + platform_assert(expected_state >= target, + "unexpected install state while sealing group %lu: " + "%lu", + target, + expected_state); platform_sleep_ns(wait); - wait = wait > 1024 ? wait : 2 * wait; - page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); + wait = wait > 2048 ? wait : 2 * wait; + continue; } - cache_lock(cc, page); - debug_assert(thread_data->addr == addr); - debug_assert(thread_data->offset >= sizeof(shard_log_hdr)); - debug_assert(thread_data->offset <= shard_log_page_size(log->cfg)); + /* The terminal claim now protects current after its hazard is cleared. */ + shard_log_reservation_slot_clear(log, tid); + shard_log_wait_for_accepting_publication(log, current); + platform_assert(current->state == SHARD_LOG_GROUP_OPEN); + current->state = SHARD_LOG_GROUP_CLOSING; + current->close = SHARD_LOG_CLOSE_STREAM; + target = current->id; + /* install.state already publishes the terminal ticket. */ + __atomic_store_n(&log->accepting.group, NULL, __ATOMIC_SEQ_CST); + break; + } - shard_log_hdr *hdr = (shard_log_hdr *)page->data; - log_entry *cursor = (log_entry *)(page->data + thread_data->offset); - uint64 free_space = shard_log_page_size(log->cfg) - thread_data->offset; - if (sizeof(log_entry) <= free_space) { - log_entry_set_terminal(cursor); - } - hdr->checksum = shard_log_checksum(log->cfg, page); + return shard_log_wait_for_ticket_to_be_durable(log, target); +} - cache_unlock(cc, page); - cache_unclaim(cc, page); - cache_unget(cc, page); +/* Final destruction, called by the thread which releases the last handle ref. + */ +static void +shard_log_destroy(shard_log *log) +{ + mini_release(&log->mini); - /* Subsequent writes must allocate a new append page. */ - thread_data->addr = SHARD_UNMAPPED; - thread_data->offset = 0; + shard_log_group *group = log->groups_head; + while (group != NULL) { + shard_log_group *next = group->next; + shard_log_group_free(log, group); + group = next; + } + group = log->reusable_pool; + while (group != NULL) { + shard_log_group *next = group->pool_next; + shard_log_group_free(log, group); + group = next; } + group = log->emergency_pool; + while (group != NULL) { + shard_log_group *next = group->pool_next; + shard_log_group_free(log, group); + group = next; + } + + platform_status rc = platform_mutex_destroy(&log->durability_lock); + platform_assert_status_ok(rc); + rc = platform_mutex_destroy(&log->graduate_lock); + platform_assert_status_ok(rc); + rc = platform_mutex_destroy(&log->group_pool_lock); + platform_assert_status_ok(rc); /* - * The stream is now immutable. Release the mini-allocator's unused - * per-batch reserve so no future allocation touches this stream, and free - * the handle. The caller already holds the stream's identity (captured at - * creation) and later frees the on-disk extents via log_dec_ref(). + * The handle's mini-allocator reference is dropped last. If the owner has + * already released the log_head, this deallocates the stream, so all staged + * page handles and unused reserve extents must be released first. */ - mini_release(&log->mini); + (void)mini_dec_ref(log->cc, log->meta_head, PAGE_TYPE_LOG); platform_free(log->heap_id, log); - return STATUS_OK; +} + +static void +shard_log_deinit(log_handle *logh) +{ + shard_log *log = (shard_log *)logh; + + for (threadid tid = 0; tid < MAX_THREADS; tid++) { + platform_assert(shard_log_reservation_slot_load(log, tid) == 0, + "log_deinit with an unconsumed operation"); + } + debug_assert( + !__atomic_exchange_n(&log->owner_released, TRUE, __ATOMIC_ACQ_REL), + "log owner reference consumed more than once"); + shard_log_handle_ref_release(log); } void -log_dec_ref(cache *cc, const log_head *segment) +shard_log_dec_ref(cache *cc, const log_head *segment) { if (segment->meta_addr == 0) { return; } - refcount ref = mini_dec_ref(cc, segment->meta_addr, PAGE_TYPE_LOG); - platform_assert(ref == 0); + /* + * The live handle holds an independent mini-allocator reference. A split + * make-durable ticket keeps that handle alive, so the owner may release the + * head while a ticket is waiting. + */ + (void)mini_dec_ref(cc, segment->meta_addr, PAGE_TYPE_LOG); } -log_head +static log_head shard_log_get_head(log_handle *logh) { shard_log *log = (shard_log *)logh; return (log_head){ .addr = log->addr, .meta_addr = log->meta_head, - .magic = log->magic, + .nonce = log->nonce, }; } -bool32 -shard_log_valid(shard_log_config *cfg, page_handle *page, uint64 magic) +static bool32 +shard_log_valid(shard_log_config *cfg, page_handle *page, log_nonce nonce) { shard_log_hdr *hdr = (shard_log_hdr *)page->data; - return hdr->magic == magic + return log_nonce_is_equal(hdr->nonce, nonce) && platform_checksum_is_equal(hdr->checksum, shard_log_checksum(cfg, page)); } -uint64 +static uint64 shard_log_next_extent_addr(shard_log_config *cfg, page_handle *page) { shard_log_hdr *hdr = (shard_log_hdr *)page->data; return hdr->next_extent_addr; } +/* The base address of the extent holding addr. */ +static uint64 +shard_log_extent_base(cache *cc, uint64 addr) +{ + return allocator_config_extent_base_addr( + allocator_get_config(cache_get_allocator(cc)), addr); +} + +/* + * Could addr be the base address of a log extent on this device? + * + * next_extent_addr is read out of a page that a crash may have left holding + * anything at all, so it is checked against the device geometry before it is + * followed; once the walk lands on a page, its nonce and checksum are what + * vouch for the contents. + * + * Deliberately not a refcount test. Crash recovery walks a stream precisely in + * order to rebuild the refcount map, so the walk must not consult the map it is + * about to populate -- and before allocator_recovery_begin() there is no map to + * consult: rc_allocator leaves it NULL until then, so asking would fault. + */ +static bool32 +shard_log_valid_extent_addr(cache *cc, shard_log_config *cfg, uint64 addr) +{ + uint64 extent_size = shard_log_extent_size(cfg); + uint64 capacity = allocator_get_capacity(cache_get_allocator(cc)); + + return addr != 0 && addr % extent_size == 0 + && addr <= capacity - extent_size; +} + +/* + * Snapshot which complete pages of an extent may be read from the backing + * store. Query every page rather than assuming a readable prefix: the current + * file backend has prefix-shaped EOF, but the abstract interface also permits + * backends with holes. + */ +static platform_status +shard_log_extent_readable_pages(cache *cc, + shard_log_config *cfg, + uint64 extent_addr, + bool32 readable_pages[MAX_PAGES_PER_EXTENT]) +{ + uint64 pages_per_extent = shard_log_pages_per_extent(cfg); + uint64 page_size = shard_log_page_size(cfg); + platform_assert(pages_per_extent <= MAX_PAGES_PER_EXTENT); + + for (uint64 i = 0; i < pages_per_extent; i++) { + uint64 page_addr = extent_addr + i * page_size; + platform_status rc = + cache_range_is_readable(cc, page_addr, page_size, &readable_pages[i]); + if (!SUCCESS(rc)) { + return rc; + } + } + return STATUS_OK; +} + +/* Issue every safe prefetch before the caller starts waiting on cache_get(). */ +static void +shard_log_prefetch_readable_pages( + cache *cc, + shard_log_config *cfg, + uint64 extent_addr, + const bool32 readable_pages[MAX_PAGES_PER_EXTENT]) +{ + uint64 pages_per_extent = shard_log_pages_per_extent(cfg); + uint64 page_size = shard_log_page_size(cfg); + bool32 all_readable = TRUE; + + for (uint64 i = 0; i < pages_per_extent; i++) { + all_readable &= readable_pages[i]; + } + if (all_readable) { + cache_prefetch(cc, extent_addr, PAGE_TYPE_LOG); + return; + } + + for (uint64 i = 0; i < pages_per_extent; i++) { + if (readable_pages[i]) { + cache_prefetch_page(cc, extent_addr + i * page_size, PAGE_TYPE_LOG); + } + } +} + +static platform_status +shard_log_recover_page_blob_allocations(cache *cc, + shard_log_config *cfg, + page_handle *page) +{ + for (log_entry *le = first_log_entry(page->data); + !terminal_log_entry(cfg, page->data, le); + le = log_entry_next(le)) + { + if (log_entry_message_is_blob(le)) { + message msg = log_entry_message(cc, le); + platform_status rc = blob_recover_allocations(cc, message_slice(msg)); + if (!SUCCESS(rc)) { + return rc; + } + } + } + return STATUS_OK; +} + +platform_status +shard_log_recover_allocations(cache *cc, shard_log_config *cfg, log_head head) +{ + if (head.addr == 0) { + return STATUS_OK; // no such log + } + + /* + * The metadata head sits in an extent of its own, allocated before the mini + * allocator that owns the data extents (see shard_log_init()), so the + * page-header chain never reaches it. It has to be recorded on its own or + * the extent is left looking free while the durable record still names it. + */ + allocator *al = cache_get_allocator(cc); + uint64 meta_base = shard_log_extent_base(cc, head.meta_addr); + platform_status rc = + allocator_recovery_record_reference(al, meta_base, PAGE_TYPE_LOG); + if (!SUCCESS(rc)) { + return rc; + } + + uint64 budget = allocator_get_capacity(al) / shard_log_extent_size(cfg); + uint64 extent_addr = head.addr; + while (shard_log_valid_extent_addr(cc, cfg, extent_addr)) { + if (budget-- == 0) { + platform_error_log("shard_log_recover_allocations: stream from %lu " + "has more extents than the device holds; its " + "next-extent chain is corrupt\n", + head.addr); + return STATUS_INVALID_STATE; + } + + /* + * Record the current extent before cache_get(): the map being rebuilt + * does not yet permit reads from it. This also records a wholly + * unreadable successor named by the final valid page, protecting the + * mini allocator's unused reserve until replay completes. + */ + rc = allocator_recovery_record_reference(al, extent_addr, PAGE_TYPE_LOG); + if (!SUCCESS(rc)) { + return rc; + } + + bool32 readable_pages[MAX_PAGES_PER_EXTENT]; + rc = + shard_log_extent_readable_pages(cc, cfg, extent_addr, readable_pages); + if (!SUCCESS(rc)) { + return rc; + } + shard_log_prefetch_readable_pages(cc, cfg, extent_addr, readable_pages); + + uint64 next_extent_addr = 0; + uint64 pages_per_extent = shard_log_pages_per_extent(cfg); + uint64 page_size = shard_log_page_size(cfg); + for (uint64 i = 0; i < pages_per_extent; i++) { + if (!readable_pages[i]) { + continue; + } + + uint64 page_addr = extent_addr + i * page_size; + page_handle *page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_LOG); + if (shard_log_valid(cfg, page, head.nonce)) { + /* The latest valid page has the newest successor link. */ + next_extent_addr = shard_log_next_extent_addr(cfg, page); + /* Include valid pages in a trailing incomplete group. */ + rc = shard_log_recover_page_blob_allocations(cc, cfg, page); + } + cache_unget(cc, page); + if (!SUCCESS(rc)) { + return rc; + } + } + extent_addr = next_extent_addr; + } + + return STATUS_OK; +} + /* * Bytes appended to the stream so far. The mini-allocator already tracks the * extents it has handed out across all of the stream's batches (data and blob), @@ -405,7 +1973,7 @@ shard_log_next_extent_addr(shard_log_config *cfg, page_handle *page) * fresh stream reports 0, so a caller comparing against a threshold cannot be * tricked into rotating a stream that has had nothing written to it. */ -uint64 +static uint64 shard_log_get_size(log_handle *logh) { shard_log *log = (shard_log *)logh; @@ -413,35 +1981,96 @@ shard_log_get_size(log_handle *logh) * shard_log_extent_size(log->cfg); } +static bool32 +shard_log_is_empty(log_handle *logh) +{ + shard_log *log = (shard_log *)logh; + return !shard_log_atomic_bool_load(&log->has_records); +} + static log_ops shard_log_ops = { - .write = shard_log_write, - .seal = shard_log_seal, - .head = shard_log_get_head, - .size = shard_log_get_size, + .write_reserve = shard_log_write_reserve, + .write_reserved = shard_log_write_reserved, + .make_durable_begin = shard_log_make_durable_begin, + .make_durable_wait = shard_log_make_durable_wait, + .seal = shard_log_seal, + .deinit = shard_log_deinit, + .head = shard_log_get_head, + .is_empty = shard_log_is_empty, + .size = shard_log_get_size, }; static platform_status -shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg) +shard_log_init(shard_log *log, + cache *cc, + shard_log_config *cfg, + platform_heap_id hid) { memset(log, 0, sizeof(shard_log)); log->cc = cc; log->cfg = cfg; + log->heap_id = hid; log->super.ops = &shard_log_ops; - uint64 magic_idx = __sync_fetch_and_add(&shard_log_magic_idx, 1); - log->magic = platform_checksum64(&magic_idx, sizeof(uint64), cfg->seed); + platform_status rc = platform_random_bytes(&log->nonce, sizeof(log->nonce)); + if (!SUCCESS(rc)) { + platform_error_log("shard_log_init: failed to generate log nonce: %s\n", + platform_status_to_string(rc)); + return rc; + } - allocator *al = cache_get_allocator(cc); - platform_status rc = allocator_alloc(al, &log->meta_head, PAGE_TYPE_LOG); - platform_assert_status_ok(rc); + rc = platform_mutex_init(&log->group_pool_lock, 0, hid); + if (!SUCCESS(rc)) { + return rc; + } + rc = platform_mutex_init(&log->graduate_lock, 0, hid); + if (!SUCCESS(rc)) { + platform_mutex_destroy(&log->group_pool_lock); + return rc; + } + rc = platform_mutex_init(&log->durability_lock, 0, hid); + if (!SUCCESS(rc)) { + platform_mutex_destroy(&log->graduate_lock); + platform_mutex_destroy(&log->group_pool_lock); + return rc; + } - for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { - shard_log_thread_data *thread_data = - shard_log_get_thread_data(log, thr_i); - thread_data->addr = SHARD_UNMAPPED; - thread_data->offset = 0; + shard_log_group *current = shard_log_group_malloc(log, FALSE); + if (current == NULL) { + platform_mutex_destroy(&log->durability_lock); + platform_mutex_destroy(&log->graduate_lock); + platform_mutex_destroy(&log->group_pool_lock); + return STATUS_NO_MEMORY; + } + current->id = SHARD_LOG_FIRST_GROUP_ID; + log->groups_head = current; + __atomic_store_n(&log->accepting.group, current, __ATOMIC_RELAXED); + __atomic_store_n(&log->accepting.id, current->id, __ATOMIC_RELAXED); + __atomic_store_n(&log->install.state, current->id, __ATOMIC_RELAXED); + + for (uint64 i = 0; i < SHARD_LOG_NUM_EMERGENCY_GROUPS; i++) { + shard_log_group *emergency = shard_log_group_malloc(log, TRUE); + if (emergency == NULL) { + shard_log_group_free(log, current); + while (log->emergency_pool != NULL) { + emergency = log->emergency_pool; + log->emergency_pool = emergency->pool_next; + emergency->pool_next = NULL; + shard_log_group_free(log, emergency); + } + platform_mutex_destroy(&log->durability_lock); + platform_mutex_destroy(&log->graduate_lock); + platform_mutex_destroy(&log->group_pool_lock); + return STATUS_NO_MEMORY; + } + emergency->pool_next = log->emergency_pool; + log->emergency_pool = emergency; } + allocator *al = cache_get_allocator(cc); + rc = allocator_alloc(al, &log->meta_head, PAGE_TYPE_LOG); + platform_assert_status_ok(rc); + log->addr = mini_init_with_types(&log->mini, cc, log->meta_head, @@ -449,37 +2078,53 @@ shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg) NUM_BLOB_BATCHES + 1, PAGE_TYPE_LOG, shard_log_page_type_table); + /* + * mini_init's initial external reference belongs to the log_head owner. + * Keep a second reference for the live handle, so handle_refs can extend + * both the in-memory and allocation lifetimes without per-ticket refcount + * traffic. + */ + (void)mini_inc_ref(cc, log->meta_head); // platform_default_log("addr: %lu meta_head: %lu\n", log->addr, // log->meta_head); // Baseline for shard_log_get_size(): the stream's fixed overhead. log->initial_extents = mini_num_extents(&log->mini); + __atomic_store_n(&log->handle_refs, 1, __ATOMIC_RELAXED); return STATUS_OK; } -log_handle * -shard_log_create(cache *cc, shard_log_config *cfg, platform_heap_id hid) +platform_status +shard_log_create(cache *cc, + shard_log_config *cfg, + platform_heap_id hid, + log_handle **log_out) { + if (log_out == NULL) { + return STATUS_BAD_PARAM; + } + *log_out = NULL; + shard_log *slog = TYPED_MALLOC(hid, slog); if (slog == NULL) { platform_error_log("shard_log_create: failed to allocate shard_log\n"); - return NULL; + return STATUS_NO_MEMORY; } - platform_status rc = shard_log_init(slog, cc, cfg); + // The heap is remembered in the log so that log_deinit() can free the handle + // and its staging buffers without the caller touching platform_free(). + platform_status rc = shard_log_init(slog, cc, cfg, hid); if (!SUCCESS(rc)) { platform_error_log("shard_log_create: shard_log_init failed: %s\n", platform_status_to_string(rc)); platform_free(hid, slog); - return NULL; + return rc; } - // Remember the heap so log_seal() can free the handle without the caller - // touching platform_free() directly. - slog->heap_id = hid; - return (log_handle *)slog; + *log_out = (log_handle *)slog; + return STATUS_OK; } -int +static int shard_log_compare(const void *p1, const void *p2, void *unused) { log_entry **le1 = (log_entry **)p1; @@ -500,7 +2145,7 @@ shard_log_compare(const void *p1, const void *p2, void *unused) return 0; } -void +static void shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) { shard_log_iterator *itor = (shard_log_iterator *)itorh; @@ -519,21 +2164,21 @@ shard_log_iterator_curr_generations(log_iterator *itorh, *leaf_generation = itor->entries[itor->pos]->leaf_generation; } -bool32 +static bool32 shard_log_iterator_can_prev(iterator *itorh) { shard_log_iterator *itor = (shard_log_iterator *)itorh; return itor->pos >= 0; } -bool32 +static bool32 shard_log_iterator_can_next(iterator *itorh) { shard_log_iterator *itor = (shard_log_iterator *)itorh; return itor->pos < itor->num_entries; } -platform_status +static platform_status shard_log_iterator_next(iterator *itorh) { shard_log_iterator *itor = (shard_log_iterator *)itorh; @@ -571,7 +2216,7 @@ shard_log_print(shard_log *log) cache *cc = log->cc; uint64 extent_addr = log->addr; shard_log_config *cfg = log->cfg; - uint64 magic = log->magic; + log_nonce nonce = log->nonce; data_config *dcfg = cfg->data_cfg; uint64 pages_per_extent = shard_log_pages_per_extent(cfg); allocator *al = cache_get_allocator(cc); @@ -582,7 +2227,7 @@ shard_log_print(shard_log *log) for (uint64 i = 0; i < pages_per_extent; i++) { uint64 page_addr = extent_addr + i * shard_log_page_size(cfg); page_handle *page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_LOG); - if (shard_log_valid(cfg, page, magic)) { + if (shard_log_valid(cfg, page, nonce)) { next_extent_addr = shard_log_next_extent_addr(cfg, page); for (log_entry *le = first_log_entry(page->data); !terminal_log_entry(cfg, page->data, le); @@ -617,7 +2262,7 @@ shard_log_iterator_deinit(log_iterator *itorh) platform_free(hid, itor); // the handle, from shard_log_iterator_create() } -const static iterator_ops shard_log_iterator_ops = { +static const iterator_ops shard_log_iterator_ops = { .curr = shard_log_iterator_curr, .can_prev = shard_log_iterator_can_prev, .can_next = shard_log_iterator_can_next, @@ -625,27 +2270,73 @@ const static iterator_ops shard_log_iterator_ops = { .print = NULL, }; -const static log_iterator_ops shard_log_log_iterator_ops = { +/* + * TRUE only when the accepted records run to a page marked as ending a sealed + * stream. FALSE for a live stream, and for one whose tail was lost -- in which + * case the records yielded are still a valid prefix, but nothing written after + * this stream may be replayed on top of them. + */ +static bool32 +shard_log_iterator_stream_complete(log_iterator *itorh) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + return itor->stream_complete; +} + +static const log_iterator_ops shard_log_log_iterator_ops = { .curr_generations = shard_log_iterator_curr_generations, .deinit = shard_log_iterator_deinit, + .stream_complete = shard_log_iterator_stream_complete, }; +static platform_status +shard_log_iterator_accept_group(shard_log_iterator *itor, + uint64 group_pages, + uint64 group_entries, + uint64 group_contents_size, + uint64 *num_valid_pages, + uint64 *contents_size) +{ + if (group_pages > UINT64_MAX - *num_valid_pages + || group_entries > UINT64_MAX - itor->num_entries + || group_contents_size > UINT64_MAX - *contents_size) + { + return STATUS_LIMIT_EXCEEDED; + } + + uint64 new_num_entries = itor->num_entries + group_entries; + uint64 new_contents_size = *contents_size + group_contents_size; + if (new_num_entries > SIZE_MAX / sizeof(*itor->entries) + || new_contents_size > SIZE_MAX) + { + return STATUS_LIMIT_EXCEEDED; + } + + *num_valid_pages += group_pages; + itor->num_entries = new_num_entries; + *contents_size = new_contents_size; + return STATUS_OK; +} + static platform_status shard_log_iterator_init(cache *cc, shard_log_config *cfg, platform_heap_id hid, uint64 addr, - uint64 magic, + log_nonce nonce, + uint64 first_needed_generation, shard_log_iterator *itor) { - page_handle *page; - uint64 i; - uint64 pages_per_extent = shard_log_pages_per_extent(cfg); - uint64 page_addr; - uint64 num_valid_pages = 0; - uint64 extent_addr; - uint64 next_extent_addr; - uint64 contents_size; + page_handle *page; + uint64 i; + uint64 pages_per_extent = shard_log_pages_per_extent(cfg); + uint64 page_addr; + uint64 num_valid_pages = 0; + uint64 extent_addr; + uint64 next_extent_addr; + uint64 contents_size = 0; + platform_status rc; + bool32 readable_pages[MAX_PAGES_PER_EXTENT]; memset(itor, 0, sizeof(shard_log_iterator)); itor->super.super.ops = &shard_log_iterator_ops; // generic iterator @@ -655,29 +2346,205 @@ shard_log_iterator_init(cache *cc, itor->cfg = cfg; allocator *al = cache_get_allocator(cc); - // traverse the log extents and calculate the required space + /* + * First pass: work out how much of the stream is replayable. + * + * Only whole groups may be replayed, and only an unbroken run of them from + * the start: a group that is intact but follows a broken one cannot be + * applied, because the records in between are missing and the result would + * not be a prefix of anything that happened. + * + * A group is intact when the page count declared by its terminator matches + * the number of its pages actually present. Later groups may already be + * staging records, but physical graduation is serialized: a group's + * terminator obtains its address before any page of the next group. A + * group's pages are therefore contiguous in the traversal and the + * replayable portion is a prefix of it. We only have to track a run at a + * time, and count pages. + */ + uint64 group_id = 0; // the run currently being tallied + // On-disk ids must run 1, 2, 3, ... with no gaps. + uint64 expect_group = SHARD_LOG_FIRST_GROUP_ID; + bool32 in_group = FALSE; + uint64 group_pages = 0; // pages of it seen + uint64 group_entries = 0; + uint64 group_contents_size = 0; + uint64 group_declared = 0; // pages its terminator claims, 0 if unseen + // whether its terminator also says the stream ends here + bool32 group_ends_stream = FALSE; + bool32 broken = FALSE; // hit a group we cannot replay + bool32 finished = FALSE; // accepted an end-of-stream group + + /* + * The refcount gate is what stops the walk, and it is load-bearing rather + * than defensive: the last page of a finished stream names a next extent + * that shard_log_deinit() then released (mini_release() drops the unused + * per-batch reserve), so the chain outlives the extent it points at. + * Reading there would break cache_get()'s rule that a page belong to an + * allocated extent. + * + * It works during crash recovery too, even though the map is rebuilt from + * scratch: the rebuild walk (shard_log_recover_allocations()) runs first and + * records the stream's extents plus its still-reachable successor reserve. + * The range bitmap below prevents reads of absent pages in that reserve (or + * in a partial extent); nonce and checksum reject readable non-log contents. + */ extent_addr = addr; - while (extent_addr != 0 && allocator_get_refcount(al, extent_addr) > 0) { - cache_prefetch(cc, extent_addr, PAGE_TYPE_LOG); + while (!broken && !finished && extent_addr != 0 + && allocator_get_refcount(al, extent_addr) > 0) + { + rc = + shard_log_extent_readable_pages(cc, cfg, extent_addr, readable_pages); + if (!SUCCESS(rc)) { + return rc; + } + shard_log_prefetch_readable_pages(cc, cfg, extent_addr, readable_pages); + next_extent_addr = 0; for (i = 0; i < pages_per_extent; i++) { page_addr = extent_addr + i * shard_log_page_size(cfg); - page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_LOG); - if (!shard_log_valid(cfg, page, magic)) { + if (!readable_pages[i]) { + continue; + } + page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_LOG); + if (!shard_log_valid(cfg, page, nonce)) { + /* + * A page that was never written, or whose write was lost. Keep + * scanning the extent rather than stopping here: the group this + * page belongs to is now short of its declared count and will be + * rejected on that basis, which is the check that matters. + */ + cache_unget(cc, page); + continue; + } + shard_log_hdr *hdr = (shard_log_hdr *)page->data; + + if (in_group && hdr->group_id != group_id) { + // The run ended; judge it before starting the next. + if (group_declared != 0 && group_pages == group_declared) { + rc = shard_log_iterator_accept_group(itor, + group_pages, + group_entries, + group_contents_size, + &num_valid_pages, + &contents_size); + if (!SUCCESS(rc)) { + cache_unget(cc, page); + return rc; + } + expect_group = group_id + 1; + itor->stream_complete = group_ends_stream; + if (group_ends_stream) { + /* EOS is authoritative: later pages are not in this stream. + */ + cache_unget(cc, page); + finished = TRUE; + break; + } + } else { + broken = TRUE; + } + in_group = FALSE; + } + if (broken) { + cache_unget(cc, page); + break; + } + if (!in_group) { + /* + * Group ids are dense, so a jump means a whole group left no trace + * on disk -- every one of its pages was lost. Its own count cannot + * report that (there is nothing left to count), so the sequence has + * to. Replaying across such a hole would skip records and produce + * a state that never existed. + */ + if (hdr->group_id != expect_group) { + platform_error_log("shard_log_iterator_init: log skips from " + "group %lu to %lu; discarding the rest\n", + expect_group, + hdr->group_id); + cache_unget(cc, page); + broken = TRUE; + break; + } + in_group = TRUE; + group_id = hdr->group_id; + group_pages = 0; + group_entries = 0; + group_contents_size = 0; + group_declared = 0; + group_ends_stream = FALSE; + } + + uint64 page_entries; + uint64 page_contents_size; + rc = shard_log_measure_page_records(cc, + cfg, + page, + first_needed_generation, + &page_entries, + &page_contents_size); + if (!SUCCESS(rc)) { + platform_error_log("shard_log_iterator_init: record validation " + "failed in group %lu at log page %lu: %s; " + "discarding this group and the rest\n", + group_id, + page_addr, + platform_status_to_string(rc)); + cache_unget(cc, page); + if (STATUS_IS_EQ(rc, STATUS_NO_MEMORY) + || STATUS_IS_EQ(rc, STATUS_LIMIT_EXCEEDED)) + { + return rc; + } + broken = TRUE; + break; + } + if (group_pages == UINT64_MAX + || page_entries > UINT64_MAX - group_entries + || page_contents_size > UINT64_MAX - group_contents_size) + { cache_unget(cc, page); - goto finished_first_pass; + return STATUS_LIMIT_EXCEEDED; + } + group_pages++; + group_entries += page_entries; + group_contents_size += page_contents_size; + if (hdr->pages_in_group != 0) { + if (group_declared != 0) { + platform_error_log("shard_log_iterator_init: group %lu has two " + "terminators; discarding it and the rest\n", + group_id); + cache_unget(cc, page); + broken = TRUE; + break; + } + group_declared = + hdr->pages_in_group & SHARD_LOG_PAGES_IN_GROUP_MASK; + group_ends_stream = + (hdr->pages_in_group & SHARD_LOG_END_OF_STREAM) != 0; } - num_valid_pages++; - itor->num_entries += ((shard_log_hdr *)page->data)->num_entries; next_extent_addr = shard_log_next_extent_addr(cfg, page); cache_unget(cc, page); } extent_addr = next_extent_addr; } + if (!broken && !finished && in_group) { + if (group_declared != 0 && group_pages == group_declared) { + rc = shard_log_iterator_accept_group(itor, + group_pages, + group_entries, + group_contents_size, + &num_valid_pages, + &contents_size); + if (!SUCCESS(rc)) { + return rc; + } + itor->stream_complete = group_ends_stream; + } + // Otherwise the stream ends in an unclosed group: discard it. + } -finished_first_pass: - - contents_size = num_valid_pages * shard_log_page_size(cfg); if (contents_size != 0) { itor->contents = TYPED_ARRAY_MALLOC(hid, itor->contents, contents_size); if (itor->contents == NULL) { @@ -698,28 +2565,57 @@ shard_log_iterator_init(cache *cc, } } - // traverse the log extents again and copy the kv pairs - log_entry *cursor = (log_entry *)itor->contents; - uint64 entry_idx = 0; - extent_addr = addr; - while (extent_addr != 0 && allocator_get_refcount(al, extent_addr) > 0) { - cache_prefetch(cc, extent_addr, PAGE_TYPE_LOG); + /* + * Second pass: copy the records out of the pages the first pass accepted. + * Those are the first num_valid_pages valid pages of the traversal, since + * the replayable portion is a prefix. + */ + char *cursor = itor->contents; + uint64 contents_copied = 0; + uint64 entry_idx = 0; + uint64 pages_taken = 0; + extent_addr = addr; + while (pages_taken < num_valid_pages && extent_addr != 0 + && allocator_get_refcount(al, extent_addr) > 0) + { + rc = + shard_log_extent_readable_pages(cc, cfg, extent_addr, readable_pages); + if (!SUCCESS(rc)) { + platform_free(hid, itor->entries); + platform_free(hid, itor->contents); + itor->entries = NULL; + itor->contents = NULL; + return rc; + } + shard_log_prefetch_readable_pages(cc, cfg, extent_addr, readable_pages); + next_extent_addr = 0; - for (i = 0; i < pages_per_extent; i++) { + for (i = 0; i < pages_per_extent && pages_taken < num_valid_pages; i++) { page_addr = extent_addr + i * shard_log_page_size(cfg); - page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_LOG); - if (!shard_log_valid(cfg, page, magic)) { + if (!readable_pages[i]) { + continue; + } + page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_LOG); + if (!shard_log_valid(cfg, page, nonce)) { cache_unget(cc, page); - goto finished_second_pass; + continue; } + pages_taken++; for (log_entry *le = first_log_entry(page->data); !terminal_log_entry(cfg, page->data, le); le = log_entry_next(le)) { - memmove(cursor, le, sizeof_log_entry(le)); - itor->entries[entry_idx] = cursor; + if (le->memtable_generation < first_needed_generation) { + continue; + } + uint64 entry_size = sizeof_log_entry(le); + platform_assert(entry_idx < itor->num_entries); + platform_assert(entry_size <= contents_size - contents_copied); + memmove(cursor, le, entry_size); + itor->entries[entry_idx] = (log_entry *)cursor; entry_idx++; - cursor = log_entry_next(cursor); + cursor += entry_size; + contents_copied += entry_size; } next_extent_addr = shard_log_next_extent_addr(cfg, page); cache_unget(cc, page); @@ -728,9 +2624,9 @@ shard_log_iterator_init(cache *cc, } debug_assert(entry_idx == itor->num_entries); + debug_assert(contents_copied == contents_size); // sort by generation -finished_second_pass: if (itor->num_entries != 0) { log_entry *tmp; platform_sort_slow(itor->entries, @@ -744,26 +2640,34 @@ shard_log_iterator_init(cache *cc, return STATUS_OK; } -log_iterator * +platform_status shard_log_iterator_create(cache *cc, shard_log_config *cfg, platform_heap_id hid, - log_head head) + log_head head, + uint64 first_needed_generation, + log_iterator **itor_out) { + if (itor_out == NULL) { + return STATUS_BAD_PARAM; + } + *itor_out = NULL; + shard_log_iterator *itor = TYPED_MALLOC(hid, itor); if (itor == NULL) { platform_error_log("shard_log_iterator_create: failed to allocate " "shard_log_iterator\n"); - return NULL; + return STATUS_NO_MEMORY; } - platform_status rc = - shard_log_iterator_init(cc, cfg, hid, head.addr, head.magic, itor); + platform_status rc = shard_log_iterator_init( + cc, cfg, hid, head.addr, head.nonce, first_needed_generation, itor); if (!SUCCESS(rc)) { platform_error_log("shard_log_iterator_create: shard_log_iterator_init " "failed: %s\n", platform_status_to_string(rc)); platform_free(hid, itor); - return NULL; + return rc; } - return &itor->super; + *itor_out = &itor->super; + return STATUS_OK; } diff --git a/src/shard_log.h b/src/shard_log.h index efc74f8e..db3c1782 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -17,6 +17,8 @@ #include "splinterdb/data.h" #include "blob_build.h" #include "mini_allocator.h" +#include "platform_mutex.h" +#include "writeback_set.h" /* * Configuration structure to set up the sharded log sub-system. @@ -26,27 +28,194 @@ typedef struct shard_log_config { data_config *data_cfg; uint64 seed; blob_build_config blob_cfg; - // data config of point message tree } shard_log_config; +typedef enum shard_log_close_mode { + SHARD_LOG_CLOSE_NONE, // an ordinary page; the group stays open + SHARD_LOG_CLOSE_GROUP, // last page of its group + SHARD_LOG_CLOSE_STREAM, // last page of its group and of the stream +} shard_log_close_mode; + +typedef enum shard_log_buffer_state { + SHARD_LOG_BUFFER_OPEN, + SHARD_LOG_BUFFER_INCACHE, +} shard_log_buffer_state; + +typedef enum shard_log_group_state { + SHARD_LOG_GROUP_OPEN, + SHARD_LOG_GROUP_CLOSING, + SHARD_LOG_GROUP_TERMINATING, + SHARD_LOG_GROUP_DURABILITY_PENDING, +} shard_log_group_state; + +/* + * Per-thread staging for log appends. + * + * A thread assembles a whole page image in `buf` and only then copies it into a + * freshly allocated log page ("graduating" it). Appending is therefore a bare + * memcpy into thread-private memory, rather than a cache_get / try_claim / + * lock / unlock / unclaim / unget round trip per record against a shared page. + * + * It also means a log page is written exactly once, when it is complete: there + * is no partially-filled page on disk to be rewritten later. + */ typedef struct shard_log_thread_data { - uint64 addr; - uint64 offset; + char *buf; // page-sized image under construction + uint64 offset; // append cursor within buf + shard_log_buffer_state state; + /* Number of permanent log pages allocated from this buffer. */ + uint64 page_count; + /* Held from cache_alloc() until writeback-set enrollment succeeds. */ + page_handle *incache_page; + /* Privately owned while this thread has a reservation for the group. */ + writeback_set wbset; } PLATFORM_CACHELINE_ALIGNED shard_log_thread_data; +/* + * A reservation or durability cut writes only its calling thread's slot. + * Cache-line separation keeps unrelated operations from bouncing the same + * line merely to publish their group-selection hazards. + */ +typedef struct shard_log_reservation_slot { + /* Conservative lower bound, refined to the selected id; zero when idle. */ + uint64 ticket; +} PLATFORM_CACHELINE_ALIGNED shard_log_reservation_slot; + +_Static_assert(sizeof(shard_log_reservation_slot) == PLATFORM_CACHELINE_SIZE, + "reservation slot must occupy exactly one cache line"); + +typedef struct shard_log_group shard_log_group; + +/* Read together by every operation, separate from the contended cut claim. */ +typedef struct shard_log_accepting_frontier { + shard_log_group *group; + uint64 id; +} PLATFORM_CACHELINE_ALIGNED shard_log_accepting_frontier; + +_Static_assert(sizeof(shard_log_accepting_frontier) == PLATFORM_CACHELINE_SIZE, + "accepting frontier must occupy exactly one cache line"); + +typedef struct shard_log_install_claim { + uint64 state; +} PLATFORM_CACHELINE_ALIGNED shard_log_install_claim; + +_Static_assert(sizeof(shard_log_install_claim) == PLATFORM_CACHELINE_SIZE, + "installation claim must occupy exactly one cache line"); + +/* The high bit turns the versioned installation claim into a terminal claim. */ +#define SHARD_LOG_FIRST_GROUP_ID 1ULL +#define SHARD_LOG_INSTALL_TERMINAL_BIT (1ULL << 63) +#define SHARD_LOG_INSTALL_ID_MASK (SHARD_LOG_INSTALL_TERMINAL_BIT - 1) + +#if SPLINTER_DEBUG +typedef enum shard_log_test_hook_event { + SHARD_LOG_TEST_SUCCESSOR_POINTER_PUBLISHED, + SHARD_LOG_TEST_ACCEPTING_HANDOFF_WAIT, +} shard_log_test_hook_event; + +typedef void (*shard_log_test_hook_fn)(void *arg, + shard_log_test_hook_event event, + uint64 group_id); +#endif + +/* + * One independently staged and written-back durability group. A cut closes + * this object and immediately installs another one for new writers; the slow + * graduation, writeback wait, and device barrier happen afterward. + */ +struct shard_log_group { + uint64 id; // on-disk id and durability ticket once installed + shard_log_group_state state; + shard_log_close_mode close; + /* Assigned once by the final-page writer from the per-thread counters. */ + uint64 page_count; + /* Atomic, set once by the first reservation in this incarnation. */ + bool32 ever_used; + bool32 emergency; // Is this group from the pool of emergency groups? + /* + * First failed append after this group was selected. A non-success value + * permanently poisons the group: it must never receive a commit terminator. + * The first error is installed atomically by a completing reservation. + */ + platform_status append_error; + + shard_log_thread_data *thread_data; + char *thread_buffers; + + shard_log_group *next; + shard_log_group *pool_next; +}; + +#define SHARD_LOG_NUM_EMERGENCY_GROUPS 2 +#define SHARD_LOG_NUM_REUSABLE_GROUPS 2 + /* * Sharded log context structure. */ typedef struct shard_log { - log_handle super; // handle to log I/O ops abstraction. - cache *cc; - shard_log_config *cfg; - platform_heap_id heap_id; - shard_log_thread_data thread_data[MAX_THREADS]; - mini_allocator mini; - uint64 addr; - uint64 meta_head; - uint64 magic; + log_handle super; // handle to log I/O ops abstraction. + cache *cc; + shard_log_config *cfg; + platform_heap_id heap_id; + mini_allocator mini; + + /* Immutable state, once the log is inited. */ + uint64 addr; + uint64 meta_head; + log_nonce nonce; + + shard_log_group *groups_head; + /* + * Current group and its id. The pointer is published before its id, and the + * id is the release marker that completes that publication. The id is also + * the group's durability ticket. id - 1 is the highest published closed + * group while group is non-NULL; group == NULL publishes the terminal cut. + * install.state is the accepting group's id at rest and its successor's id + * while that successor is being installed. Threads claim installation by + * CASing install to the next group's id. A thread may claim the next + * installation after seeing the new pointer but must wait for accepting.id + * to catch up before publishing. Once the terminal cut wins the same claim, + * its high bit remains set and its low bits name the final group. + */ + shard_log_accepting_frontier accepting; + shard_log_install_claim install; + shard_log_reservation_slot reservation_slots[MAX_THREADS]; +#if SPLINTER_DEBUG + /* Optional deterministic scheduling hook for state-machine tests. */ + shard_log_test_hook_fn test_hook; + void *test_hook_arg; +#endif + + shard_log_group *emergency_pool; + /* Protects both pools and reusable_pool_count. */ + platform_mutex group_pool_lock; + shard_log_group *reusable_pool; + uint64 reusable_pool_count; + + /* Mmonotonically published group frontiers. */ + uint64 graduated_ticket; + uint64 durable_ticket; + + /* Includes one owner ref plus one per successful durability begin. + */ + uint64 handle_refs; + /* Atomic diagnostic: the owner has consumed its handle reference. */ +#if SPLINTER_DEBUG + bool32 owner_released; +#endif + /* Stream-wide: set once after the first record is staged. */ + bool32 has_records; + /* + * graduate_lock serializes group state transitions and protects groups_head + * against durable-prefix detachment. durability_lock serializes durable + * publication and reclamation. This preserves physical group order while + * allowing later groups to stage records and issue their writebacks while + * an earlier group waits at the device. Neither is held while allocating or + * waiting for cache I/O. + */ + platform_mutex graduate_lock; + platform_mutex durability_lock; + /* * Extents the mini-allocator held once the stream was initialized -- its * fixed per-stream overhead (a metadata extent plus one per batch). @@ -58,6 +227,18 @@ typedef struct shard_log { typedef struct log_entry log_entry; +/* + * Flag bit stolen from the top of shard_log_hdr::pages_in_group, marking the + * final group of a sealed stream. + * + * Without it, a stream that lost a whole trailing group is indistinguishable + * from one that simply ended: every group present is intact and contiguous, so + * nothing on disk says more was supposed to follow. Replaying up to that point + * and then moving on to the next log would skip the missing records. + */ +#define SHARD_LOG_END_OF_STREAM (1u << 31) +#define SHARD_LOG_PAGES_IN_GROUP_MASK (SHARD_LOG_END_OF_STREAM - 1) + typedef struct shard_log_iterator { log_iterator super; // IS-A log_iterator IS-A generic iterator platform_heap_id heap_id; @@ -67,6 +248,8 @@ typedef struct shard_log_iterator { log_entry **entries; uint64 num_entries; uint64 pos; + // Whether the replayable records run all the way to an end-of-stream marker. + bool32 stream_complete; } shard_log_iterator; /* @@ -77,30 +260,109 @@ typedef struct shard_log_iterator { */ typedef struct ONDISK shard_log_hdr { checksum128 checksum; - uint64 magic; + log_nonce nonce; uint64 next_extent_addr; - uint16 num_entries; + /* + * The group this page belongs to. A group is the unit of replay: either all + * of its pages are present and it is replayed, or it is discarded whole. + * That is what lets recovery reconstruct a prefix of the writes rather than + * an arbitrary subset, which contiguity of the generation tags cannot + * establish (splits advance generations without emitting a record). + */ + uint64 group_id; + /* + * Non-zero only on the group's final page, giving its total page count, with + * SHARD_LOG_END_OF_STREAM set when this is also the end of a sealed stream. + * Zero on every earlier page. The final page normally carries packed data; + * an empty group uses an empty final page. + * + * The count cannot be stamped on earlier pages because the final group size + * is not known yet. Close therefore reserves one mutable thread buffer for + * the final page and graduates it only after every ordinary page has a + * permanent address. A failed close retries frozen images without modifying + * or duplicating them. + */ + uint32 pages_in_group; + uint16 num_entries; } shard_log_hdr; /* - * Create a fresh sharded write-ahead log stream. Returns an abstract - * log_handle (or NULL on failure) to be driven through the log.h interface and - * retired with log_seal(). + * Create a fresh sharded write-ahead log stream. On success, returns an + * abstract log_handle through `log_out` to be driven through the log.h + * interface and released with log_deinit(). */ -log_handle * -shard_log_create(cache *cc, shard_log_config *cfg, platform_heap_id hid); +platform_status +shard_log_create(cache *cc, + shard_log_config *cfg, + platform_heap_id hid, + log_handle **log_out); /* * Create an iterator over the sharded log identified by `head`, reading its - * records in generation order. Returns an abstract log_iterator (or NULL on - * failure) to be driven through the log.h interface and freed with - * log_iterator_deinit(). + * records at or above `first_needed_generation` in generation order. Older + * records are already represented by the checkpoint root, so they are neither + * returned nor have their blob checksums validated. Returns an abstract + * log_iterator through `itor_out` to be driven through the log.h interface and + * freed with log_iterator_deinit(). */ -log_iterator * +platform_status shard_log_iterator_create(cache *cc, shard_log_config *cfg, platform_heap_id hid, - log_head head); + log_head head, + uint64 first_needed_generation, + log_iterator **itor_out); + +/* + * Release a stream identified by its log_head: drop the owner's reference from + * its metadata head. This normally frees the stream's on-disk extents. The live + * handle owns an independent reference, and a split-phase durability ticket + * may keep that handle alive until its matching wait, so this release is not + * required to be the final reference. Takes no handle -- the owner has called + * log_deinit() and retained only the head captured at creation. + * + * Do not use this for a stream left behind by a crash: its mini-allocator + * metadata was not made durable. Crash recovery rebuilds the allocator map + * through shard_log_recover_allocations() instead. + */ +void +shard_log_dec_ref(cache *cc, const log_head *head); + +/* + * ---- Recovering a stream's extents ---- + * + * A shard log's mini-allocator metadata is deliberately never made durable -- + * keeping it safe to read after a crash would cost a write per allocation, and + * nothing in normal operation needs it. Crash recovery therefore walks the + * stream itself, following the next-extent links in its page headers, to + * reconstruct the allocator references. It queries which individual backing + * pages are readable before issuing cache reads, so a partly written final + * extent and backends with holes need no process-wide relaxed-read mode. + * + * Record one allocator reference for every extent the durable stream can still + * reach. Call between allocator_recovery_begin() and + * allocator_recovery_finish(). A zero head.addr (an absent log slot) is not an + * error and records nothing. + * The initial extent and a wholly unreadable linked successor are still + * recorded: both remain reachable from the durable log identity during replay, + * so neither may be reused until the root-only rebuild drops the log. + * + * The metadata head is recorded first. Each stream extent is recorded before + * any of its pages are read, then every individually valid page is scanned in + * that same pass to recover the separate storage of each blob it names, + * including pages in a trailing incomplete group. Replay validates those blobs + * before it can decide that the group is incomplete, and cache reads require + * their extents not to have been reused meanwhile. Blob recovery therefore has + * to precede any replay allocation; the conservative suffix-only references + * disappear in the root-only rebuild below. + * + * The recovered references need no matching release pass. After replay is + * folded into the tree, recovery publishes a root naming no logs and rebuilds + * the allocator map again from that root. The old streams are freed by being + * absent from the second map. + */ +platform_status +shard_log_recover_allocations(cache *cc, shard_log_config *cfg, log_head head); void shard_log_config_init(shard_log_config *log_cfg, diff --git a/src/splinterdb.c b/src/splinterdb.c index 653f4bd4..7a5746b2 100644 --- a/src/splinterdb.c +++ b/src/splinterdb.c @@ -166,12 +166,23 @@ splinterdb_config_set_defaults(splinterdb_config *cfg) } if (!cfg->checkpoint_log_size_bytes) { /* - * Checkpoint once the log has grown by a cache's worth: replaying much - * more log than the cache can hold gains little, since those pages cannot - * stay resident anyway. + * Arm a checkpoint once the log has grown by a cache's worth: replaying + * much more log than the cache can hold gains little, since those pages + * cannot stay resident anyway. */ cfg->checkpoint_log_size_bytes = cfg->cache_size; } + if (!cfg->checkpoint_log_grace_bytes) { + /* + * Give a newly armed checkpoint one physical memtable budget in which to + * catch a natural rotation. The memtable space budget is twice its + * configured logical capacity; saturate so the eventual hard log limit + * cannot wrap. + */ + cfg->checkpoint_log_grace_bytes = cfg->memtable_capacity > UINT64_MAX / 2 + ? UINT64_MAX + : 2 * cfg->memtable_capacity; + } } static platform_status @@ -320,6 +331,7 @@ splinterdb_init_config(const splinterdb_config *kvs_cfg, // IN cfg.prefetch_budget, cfg.use_log, cfg.checkpoint_log_size_bytes, + cfg.checkpoint_log_grace_bytes, cfg.use_stats, FALSE, Platform_default_log_handle); @@ -576,14 +588,17 @@ splinterdb_open(const splinterdb_config *cfg, // IN * Platform heap memory is also destroyed when closing SplinterDB. * * Results: - * None. + * 0 when all acknowledged data is recoverable (possibly after replay or + * allocator reconstruction). Without force, an error closes nothing and + * leaves *kvs_in open and usable. With force, an error means preservation + * could not be guaranteed, but teardown still completes. * * Side effects: - * None. + * On success or any forced close, *kvs_in is freed and set to NULL. *----------------------------------------------------------------------------- */ -void -splinterdb_close(splinterdb **kvs_in) // IN +int +splinterdb_close(splinterdb **kvs_in, bool32 force) // IN { splinterdb *kvs = *kvs_in; platform_assert(kvs != NULL); @@ -598,18 +613,30 @@ splinterdb_close(splinterdb **kvs_in) // IN * order when these sub-systems were init'ed when a Splinter device was * created or re-opened. Otherwise, asserts will trip. */ - platform_status status = core_unmount(&kvs->spl); + platform_status status = core_unmount(&kvs->spl, force); + /* + * Every non-forced error is a refusal made before destructive teardown. + * Keep the wrapper and all of its subsystems intact so the caller can retry + * or force. A forced close always spends the handle, even when its status + * says that preservation could not be guaranteed. + */ + if (!SUCCESS(status) && !force) { + platform_error_log("SplinterDB remains open because it could not be " + "closed with guaranteed data preservation: %s\n", + platform_status_to_string(status)); + return platform_status_to_int(status); + } if (!SUCCESS(status)) { - platform_error_log("Failed to close SplinterDB instance cleanly: %s\n", + platform_error_log("SplinterDB was forcibly closed, but data " + "preservation could not be guaranteed: %s\n", platform_status_to_string(status)); } io_wait_all(kvs->io_handle); clockcache_deinit(&kvs->cache_handle); /* - * core_unmount() already persisted the refcount map and published the - * superblock's allocation state (or, on failure, deliberately left it - * invalid so the next open rebuilds); the allocator now only tears down its - * in-memory structures. + * core_unmount() either published a trustworthy refcount map or deliberately + * left allocation state invalid for recovery; the allocator now only tears + * down its in-memory structures. */ rc_allocator_deinit(&kvs->allocator_handle); task_system_deinit(&kvs->task_sys); @@ -623,6 +650,12 @@ splinterdb_close(splinterdb **kvs_in) // IN platform_heap_destroy(&heap_id); } *kvs_in = (splinterdb *)NULL; + + /* + * The instance is gone either way; a non-OK status here reports what the + * unmount could not guarantee about the data, not a failure to close. + */ + return platform_status_to_int(status); } void @@ -819,6 +852,18 @@ splinterdb_optimize(splinterdb *kvs, return 0; } +int +splinterdb_durable_barrier(splinterdb *kvs) +{ + int rc = splinterdb_ensure_thread_registered(); + if (rc != 0) { + return rc; + } + + platform_assert(kvs != NULL); + return platform_status_to_int(core_durable_barrier(&kvs->spl)); +} + struct splinterdb_iterator { core_range_iterator sri; platform_status last_rc; diff --git a/src/superblock.h b/src/superblock.h index 16922a32..c3101b53 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -37,24 +37,26 @@ #include "allocator.h" #include "platform_io.h" #include "util.h" +#include "log_data.h" #define SUPERBLOCK_FORMAT_MAGIC (0x5344425355504552ULL) // SDBSUPER -/* v2 added superblock_log_head.start_generation. */ -#define SUPERBLOCK_FORMAT_VERSION (2) +/* + * v2 added start_generation; v3 widened log identity to a 128-bit nonce; v4 + * moved the checksum and uint16 format into the fixed blob header; v5 made + * log group ids start at 1 so they equal their durability tickets. + */ +#define SUPERBLOCK_FORMAT_VERSION (5) /* The two physical superblock copies live at pages 0 and 1. */ #define SUPERBLOCK_NUM_SLOTS (2) /* - * A log's on-disk head, plus the range of memtable generations it covers. The - * addr/meta_addr/magic triple mirrors log_head's layout; the superblock stores - * it opaquely and does not depend on the log module. meta_addr == 0 means "no - * log present". + * A log's shared on-disk head, plus the range of memtable generations it + * covers. The superblock stores the head opaquely and does not depend on the + * log implementation. head.meta_addr == 0 means "no log present". */ typedef struct ONDISK superblock_log_head { - uint64 addr; - uint64 meta_addr; - uint64 magic; + log_head head; /* * First memtable generation whose entries this log received. A log's * coverage ends where the next log's begins, so the sealed log covers @@ -66,8 +68,17 @@ typedef struct ONDISK superblock_log_head { uint64 start_generation; } superblock_log_head; -/* An empty (absent) log slot: meta_addr == 0. */ -#define SUPERBLOCK_NO_LOG(info) ((info).meta_addr == 0) +/* Embedding log_head preserves the v3 addr/meta_addr/nonce byte layout. */ +_Static_assert(offsetof(superblock_log_head, head) == 0, + "log head must begin the superblock log descriptor"); +_Static_assert(offsetof(superblock_log_head, start_generation) + == sizeof(log_head), + "superblock log generation layout changed"); +_Static_assert(sizeof(superblock_log_head) == sizeof(log_head) + sizeof(uint64), + "superblock log descriptor layout changed"); + +/* An empty (absent) log slot: head.meta_addr == 0. */ +#define SUPERBLOCK_NO_LOG(info) ((info).head.meta_addr == 0) /* * The durable per-tree record. The instance always has exactly one tree (from diff --git a/src/trunk.c b/src/trunk.c index aa1eb730..7bdc28a9 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -1742,11 +1742,11 @@ bundle_dec_all_refs(trunk_context *context, bundle *bndl) // cache_unget(context->cc, page); // } -static void +static platform_status trunk_ondisk_node_dec_ref(trunk_context *context, uint64 addr); /* Prerequisite: addr must be in the AL_NO_REFS state. */ -static void +static platform_status trunk_ondisk_node_gc(trunk_context *context, uint64 addr) { trunk_node node; @@ -1754,8 +1754,12 @@ trunk_ondisk_node_gc(trunk_context *context, uint64 addr) if (SUCCESS(rc)) { if (!trunk_node_is_leaf(&node)) { for (uint64 i = 0; i < vector_length(&node.pivots) - 1; i++) { - trunk_pivot *pvt = vector_get(&node.pivots, i); - trunk_ondisk_node_dec_ref(context, pvt->child_addr); + trunk_pivot *pvt = vector_get(&node.pivots, i); + platform_status child_rc = + trunk_ondisk_node_dec_ref(context, pvt->child_addr); + if (SUCCESS(rc) && !SUCCESS(child_rc)) { + rc = child_rc; + } } } for (uint64 i = 0; i < vector_length(&node.pivot_bundles); i++) { @@ -1775,6 +1779,7 @@ trunk_ondisk_node_gc(trunk_context *context, uint64 addr) } cache_extent_discard(context->cc, addr, PAGE_TYPE_TRUNK); allocator_dec_ref(context->al, addr, PAGE_TYPE_TRUNK); + return rc; } static void @@ -1791,8 +1796,16 @@ pending_gcs_unlock(trunk_context *context) __sync_lock_release(&context->pending_gcs_lock); } - static void +trunk_record_allocator_cleanup_error(trunk_context *context, platform_status rc) +{ + if (!SUCCESS(rc)) { + (void)__sync_bool_compare_and_swap( + &context->allocator_cleanup_status.r, STATUS_OK.r, rc.r); + } +} + +static platform_status trunk_ondisk_node_dec_ref(trunk_context *context, uint64 addr) { refcount ref = allocator_dec_ref(context->al, addr, PAGE_TYPE_TRUNK); @@ -1804,7 +1817,7 @@ trunk_ondisk_node_dec_ref(trunk_context *context, uint64 addr) "leak some disk space.", __func__, __LINE__); - return; + return STATUS_NO_MEMORY; } pgc->addr = addr; pgc->next = NULL; @@ -1820,9 +1833,10 @@ trunk_ondisk_node_dec_ref(trunk_context *context, uint64 addr) pending_gcs_unlock(context); } else { - trunk_ondisk_node_gc(context, addr); + return trunk_ondisk_node_gc(context, addr); } } + return STATUS_OK; } static void @@ -1859,6 +1873,166 @@ trunk_node_inc_all_refs(trunk_context *context, trunk_node *node) } } +/* + * ----------------------------------------------------------------------------- + * Crash-recovery allocator-reference rebuild. + * + * Reconstruct every allocator reference the tree at a durable root holds, + * without consulting the refcount map -- the map is what this is rebuilding, + * and before allocator_recovery_begin() it does not even exist. + * + * Two constraints shape the walk. + * + * The first is that reading a node requires having already recorded a reference + * to it. cache_get() insists a page belong to an allocated extent, and to a + * map under construction an extent counts as allocated only once something has + * recorded a reference to it. So each node records all of its children before + * descending into any of them. + * + * The second is that the map doubles as the set of things already visited, + * which is what makes a separate visited set unnecessary. A zero refcount + * means nothing has reached an item yet and its interior still has to be + * enumerated; a nonzero one means something already did that, and all a further + * reference adds is multiplicity. Sharing is routine here -- a flush hands one + * branch to every child it pushes down to. + * ----------------------------------------------------------------------------- + */ + +/* + * The references one bundle holds: its maplet, and each of its branches. + * Mirrors bundle_inc_all_refs(), including its rule that a null maplet is not a + * reference to anything. + */ +static platform_status +trunk_recover_bundle_refs(const trunk_context *context, bundle *bndl) +{ + if (!routing_filters_equal(&bndl->maplet, &NULL_ROUTING_FILTER)) { + platform_status rc = + routing_filter_recover_allocations(context->cc, &bndl->maplet); + if (!SUCCESS(rc)) { + return rc; + } + } + + page_type type = bundle_branch_type(bndl); + for (uint64 i = 0; i < vector_length(&bndl->branches); i++) { + branch_ref bref = vector_get(&bndl->branches, i); + platform_status rc = btree_recover_allocations( + context->cc, context->cfg->btree_cfg, branch_ref_addr(bref), type); + if (!SUCCESS(rc)) { + return rc; + } + } + return STATUS_OK; +} + +static platform_status +trunk_recover_node_refs(const trunk_context *context, uint64 addr); + +/* + * The references the subtree rooted at addr holds. A reference for addr itself + * must already have been recorded -- see the section comment -- which is why + * the caller records it and each level records its children. + */ +static platform_status +trunk_recover_node_refs(const trunk_context *context, uint64 addr) +{ + trunk_node node; + platform_status rc = trunk_node_deserialize(context, addr, &node); + if (!SUCCESS(rc)) { + platform_error_log("trunk_recover_node_refs: cannot read node %lu: %s\n", + addr, + platform_status_to_string(rc)); + return rc; + } + + uint64 num_children = + trunk_node_is_leaf(&node) ? 0 : vector_length(&node.pivots) - 1; + + for (uint64 i = 0; i < num_children && SUCCESS(rc); i++) { + trunk_pivot *pvt = vector_get(&node.pivots, i); + /* + * Sampled before recording, because recording is what makes a child look + * visited. Only the reference that discovers a child descends into it: + * going down twice would count everything beneath it twice, and a node + * holds one reference to its branches however many parents it has. + */ + bool32 unvisited = + allocator_get_refcount(context->al, pvt->child_addr) == AL_FREE; + rc = allocator_recovery_record_reference( + context->al, pvt->child_addr, PAGE_TYPE_TRUNK); + // Legal now, and only now, that the child has a reference. + if (SUCCESS(rc) && unvisited) { + rc = trunk_recover_node_refs(context, pvt->child_addr); + } + } + + uint64 num_pivot_bundles = vector_length(&node.pivot_bundles); + for (uint64 i = 0; i < num_pivot_bundles && SUCCESS(rc); i++) { + rc = trunk_recover_bundle_refs(context, + vector_get_ptr(&node.pivot_bundles, i)); + } + /* + * From 0, not from trunk_node_first_live_inflight_bundle(): serialization + * writes only the live bundles, so every bundle a node read back from disk + * holds is one it references. trunk_ondisk_node_gc() drops them the same + * way. + */ + for (uint64 i = 0; i < vector_length(&node.inflight_bundles) && SUCCESS(rc); + i++) + { + rc = trunk_recover_bundle_refs(context, + vector_get_ptr(&node.inflight_bundles, i)); + } + + trunk_node_deinit(&node, context); + return rc; +} + +platform_status +trunk_recover_allocations(const trunk_config *cfg, + cache *cc, + platform_heap_id hid, + uint64 root_addr) +{ + if (root_addr == 0) { + return STATUS_OK; // nothing has ever been incorporated + } + + /* + * Taken from the cache rather than accepted as a parameter: the branch and + * filter walks reach the allocator through the cache (mini_recover_ + * references()), so accepting a second one would just create something for + * them to disagree with. + */ + allocator *al = cache_get_allocator(cc); + + /* + * Just enough context for the read path: deserialization needs the cache, + * and the walk needs the btree config to turn a branch address into a + * metadata head. A real trunk_context cannot exist yet -- building one + * takes a reference on the root, and there is no map yet to take it in. + */ + trunk_context context = { + .cfg = cfg, + .cc = cc, + .al = al, + .hid = hid, + }; + + // The reference the durable record itself holds on the root. + platform_status rc = + allocator_recovery_record_reference(al, root_addr, PAGE_TYPE_TRUNK); + if (!SUCCESS(rc)) { + platform_error_log("trunk_recover_allocations: cannot record the root " + "%lu: %s\n", + root_addr, + platform_status_to_string(rc)); + return rc; + } + return trunk_recover_node_refs(&context, root_addr); +} + static void trunk_ondisk_node_ref_inc(const ondisk_ref *ref) { @@ -1872,7 +2046,8 @@ trunk_ondisk_node_ref_dec(const ondisk_ref *ref) { trunk_context *context = (trunk_context *)ref->arg; debug_assert(ref->type == PAGE_TYPE_TRUNK); - trunk_ondisk_node_dec_ref(context, ref->addr); + platform_status rc = trunk_ondisk_node_dec_ref(context, ref->addr); + trunk_record_allocator_cleanup_error(context, rc); } static trunk_ondisk_node_ref * @@ -2588,8 +2763,7 @@ trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) */ uint64 root_addr = snapshot->root_addr; snapshot->root_addr = 0; - trunk_ondisk_node_dec_ref(context, root_addr); - return STATUS_OK; + return trunk_ondisk_node_dec_ref(context, root_addr); } platform_status @@ -2645,7 +2819,8 @@ perform_pending_gcs(trunk_context *context) pending_gc *pgc = context->pending_gcs; while (pgc && !cache_in_use(context->cc, pgc->addr)) { - trunk_ondisk_node_gc(context, pgc->addr); + platform_status rc = trunk_ondisk_node_gc(context, pgc->addr); + trunk_record_allocator_cleanup_error(context, rc); pending_gc *next = pgc->next; platform_free(context->hid, pgc); pgc = next; @@ -6805,7 +6980,7 @@ trunk_context_init(trunk_context *context, return STATUS_OK; } -void +platform_status trunk_context_deinit(trunk_context *context) { platform_assert(context->pivot_states.num_states == 0); @@ -6819,6 +6994,7 @@ trunk_context_deinit(trunk_context *context) if (context->stats) { platform_free(context->hid, context->stats); } + return context->allocator_cleanup_status; } /************************************ diff --git a/src/trunk.h b/src/trunk.h index 70786bec..40ef0336 100644 --- a/src/trunk.h +++ b/src/trunk.h @@ -171,7 +171,9 @@ typedef struct trunk_context { uint64 pending_gcs_lock; pending_gc *pending_gcs; pending_gc *pending_gcs_tail; - incorporation_tasks tasks; + /* Sticky first error from releasing allocator references during tree GC. */ + platform_status allocator_cleanup_status; + incorporation_tasks tasks; } trunk_context; /* @@ -222,7 +224,11 @@ trunk_context_init(trunk_context *context, task_system *ts, trunk_snapshot snapshot); -void +/* + * Release the context and report any incomplete allocator-reference cleanup + * observed either here or by earlier deferred tree GC. + */ +platform_status trunk_context_deinit(trunk_context *context); /* Capture an owned reference to the current COW root without reading it. */ @@ -241,10 +247,34 @@ trunk_snapshot_create_from_addr(allocator *al, uint64 root_addr, trunk_snapshot *snapshot); -/* Drop an owned snapshot reference that was not published. */ +/* + * Drop an owned snapshot reference. The snapshot is consumed even when + * cleanup below the root cannot be completed; such an error means allocator + * accounting may conservatively retain references and must be rebuilt before + * it is persisted. + */ platform_status trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot); +/* + * Rebuild every allocator reference the tree at root_addr holds, for crash + * recovery. Consults no refcounts: this is what populates them, so it must run + * between allocator_recovery_begin() and allocator_recovery_finish(), before + * any trunk_context or snapshot exists (creating either would take a reference, + * and there is nowhere yet to take it). Hence the loose parameters rather than + * a context. A null root_addr (0) is not an error and records nothing. + * + * Records the one reference the durable record itself holds on the root, so a + * caller that goes on to take a live reference + * (trunk_snapshot_create_from_addr) ends up exactly where a clean mount's + * loaded map would have put it. + */ +platform_status +trunk_recover_allocations(const trunk_config *cfg, + cache *cc, + platform_heap_id hid, + uint64 root_addr); + /******************************** * Mutations ********************************/ diff --git a/src/writeback_set.c b/src/writeback_set.c new file mode 100644 index 00000000..f2cc5666 --- /dev/null +++ b/src/writeback_set.c @@ -0,0 +1,205 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * writeback_set.c -- + * + * Implementation of the writeback set. See writeback_set.h. + */ + +#include "platform.h" +#include "writeback_set.h" +#include "poison.h" + +void +writeback_set_init(writeback_set *set, cache *cc, platform_heap_id hid) +{ + set->cc = cc; + vector_init(&set->entries, hid); +} + +void +writeback_set_deinit(writeback_set *set) +{ + vector_deinit(&set->entries); + set->cc = NULL; +} + +void +writeback_set_reset(writeback_set *set) +{ + vector_truncate(&set->entries, 0); +} + +uint64 +writeback_set_num_requests(const writeback_set *set) +{ + return vector_length(&set->entries); +} + +/* + * Reserve room for one more member before issuing its write. + * + * The order matters: if we issued first and then failed to grow the vector, the + * write would be in flight with nothing recording it, so writeback_set_wait() + * would return without covering it and the caller would believe a page was + * durable when it had not even been waited for. Growing first means the + * subsequent append cannot fail. + */ +static platform_status +writeback_set_reserve_one(writeback_set *set) +{ + return vector_ensure_capacity(&set->entries, + vector_length(&set->entries) + 1); +} + +platform_status +writeback_set_add_page(writeback_set *set, page_handle *page, page_type type) +{ + platform_status rc = writeback_set_reserve_one(set); + if (!SUCCESS(rc)) { + return rc; + } + + writeback_set_entry entry = {.type = type}; + rc = cache_writeback_page(set->cc, page, type, &entry.request); + entry.needs_retry = !SUCCESS(rc); + + rc = vector_append(&set->entries, entry); + platform_assert_status_ok(rc); // reserved above + return STATUS_OK; +} + +platform_status +writeback_set_add_extent(writeback_set *set, uint64 addr, page_type type) +{ + platform_status rc = writeback_set_reserve_one(set); + if (!SUCCESS(rc)) { + return rc; + } + + writeback_set_entry entry = {.type = type}; + rc = cache_writeback_extent(set->cc, addr, type, &entry.request); + entry.needs_retry = !SUCCESS(rc); + + rc = vector_append(&set->entries, entry); + platform_assert_status_ok(rc); // reserved above + return STATUS_OK; +} + +platform_status +writeback_set_wait(writeback_set *set) +{ + platform_status result = STATUS_OK; + + for (uint64 i = 0; i < vector_length(&set->entries); i++) { + writeback_set_entry *entry = vector_get_ptr(&set->entries, i); + const cache_writeback_request *req = &entry->request; + + if (entry->needs_retry && SUCCESS(result)) { + /* Drain anything partially issued, but do not report the set ready. */ + result = STATUS_BUSY; + } + + while (TRUE) { + cache_writeback_status status = + cache_writeback_get_status(set->cc, req); + + if (status == CACHE_WRITEBACK_PENDING) { + /* + * cache_cleanup() reaps completions on this thread, which is what + * makes this loop progress rather than spin waiting for another + * thread to do it. + */ + cache_cleanup(set->cc); + continue; + } + + if (status == CACHE_WRITEBACK_FAILED) { + /* + * Record it but keep going, so that when we return no write + * belonging to this set is still outstanding and the caller can + * safely act on the failure. + */ + platform_error_log("writeback_set_wait: writeback of addr %lu " + "failed\n", + req->addr); + entry->needs_retry = TRUE; + result = STATUS_IO_ERROR; + } else if (status == CACHE_WRITEBACK_REDIRTIED) { + /* + * The contents we asked to be written did reach the device, so the + * request is satisfied -- but somebody dirtied the page again while + * we were writing it, which for a caller that owns these pages is a + * bug in its own locking rather than something the cache did. + */ + platform_error_log("writeback_set_wait: addr %lu was re-dirtied " + "during writeback\n", + req->addr); + entry->needs_retry = TRUE; + if (SUCCESS(result)) { + result = STATUS_BUSY; + } + } + break; + } + } + + return result; +} + +platform_status +writeback_set_retry_incomplete(writeback_set *set) +{ + platform_status result = STATUS_OK; + + for (uint64 i = 0; i < vector_length(&set->entries); i++) { + writeback_set_entry *entry = vector_get_ptr(&set->entries, i); + cache_writeback_status status = + cache_writeback_get_status(set->cc, &entry->request); + if (!entry->needs_retry && status != CACHE_WRITEBACK_FAILED + && status != CACHE_WRITEBACK_REDIRTIED) + { + continue; + } + + cache_writeback_request retry_request = entry->request; + platform_status rc; + if (entry->request.is_extent) { + rc = cache_writeback_extent( + set->cc, entry->request.addr, entry->type, &retry_request); + } else { + page_handle *page = + cache_get(set->cc, entry->request.addr, TRUE, entry->type); + if (page == NULL) { + rc = STATUS_IO_ERROR; + } else { + rc = + cache_writeback_page(set->cc, page, entry->type, &retry_request); + cache_unget(set->cc, page); + } + } + /* + * Even a failed extent retry may have issued a subset of its pages. The + * new receipt is therefore the one wait() must drain; needs_retry keeps + * the missing subset from being forgotten on the next attempt. + */ + entry->request = retry_request; + entry->needs_retry = !SUCCESS(rc); + if (!SUCCESS(rc)) { + platform_error_log( + "writeback_set_retry_incomplete: retry of addr %lu failed: %s\n", + entry->request.addr, + platform_status_to_string(rc)); + result = rc; + } + } + + return result; +} + +platform_status +writeback_set_make_durable(writeback_set *set) +{ + return cache_durable_barrier(set->cc); +} diff --git a/src/writeback_set.h b/src/writeback_set.h new file mode 100644 index 00000000..4cdfdd35 --- /dev/null +++ b/src/writeback_set.h @@ -0,0 +1,122 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * writeback_set.h -- + * + * A set of pages and extents to be written back together and then made + * durable as a unit. + * + * The cache can issue a writeback and later tell you whether it completed + * (cache_writeback_page(), cache_writeback_get_status()), but it deals in + * one page at a time. Making a group of pages durable means issuing all of + * their writes, waiting for all of them, and then taking one barrier -- in + * that order, because the barrier is what costs, and it is O(1) however + * many pages precede it. This module holds the receipts in between. + * + * The two phases are deliberately separate calls. Issuing every write + * before waiting for any is what lets the I/Os pipeline; an interface that + * let a caller interleave them would quietly serialize the group at device + * latency per page. + * + * Completion and durability are likewise separate: writeback_set_wait() + * establishes that the writes reached the device, and only + * writeback_set_make_durable() makes them survive power loss. Splitting + * them keeps the durability boundary visible at the call site rather than + * buried inside a function that mostly does something else. + * + * This module knows nothing about any particular cache implementation; it + * is written entirely against cache.h. + */ + +#pragma once + +#include "cache.h" +#include "vector.h" + +typedef struct writeback_set_entry { + cache_writeback_request request; + page_type type; + /* The request could not cover every dirty page and must be reissued. */ + bool32 needs_retry; +} writeback_set_entry; + +typedef VECTOR(writeback_set_entry) writeback_set_entry_vector; + +typedef struct writeback_set { + cache *cc; + writeback_set_entry_vector entries; +} writeback_set; + +/* Prepare an empty set. Allocates nothing until the first add. */ +void +writeback_set_init(writeback_set *set, cache *cc, platform_heap_id hid); + +/* Release the set's memory. Does not wait for any outstanding writes: a caller + * that abandons a set without waiting leaves its writes in flight, which is + * safe but means it learns nothing about whether they landed. */ +void +writeback_set_deinit(writeback_set *set); + +/* + * Issue writeback of a page (or of every page of an extent) and add it to the + * set. Non-blocking. + * + * Once the vector reservation succeeds, the member is enrolled even if it is + * temporarily locked or claimed. Such an entry records that it needs a retry, + * allowing the caller to finish assembling the set without losing either a + * partially issued extent request or the identity of a page still needing a + * write. Consequently these calls fail only if the member could not be + * enrolled at all (normally allocation failure). + */ +platform_status +writeback_set_add_page(writeback_set *set, page_handle *page, page_type type); + +platform_status +writeback_set_add_extent(writeback_set *set, uint64 addr, page_type type); + +/* + * Wait until every member's write has completed. Polls rather than blocking on + * a condition: the poll reaps I/O completions on the calling thread, so it is + * doing the work rather than merely waiting for someone else to. + * + * This is completion, NOT durability -- follow with + * writeback_set_make_durable(). + * + * Returns STATUS_IO_ERROR if a write failed, or STATUS_BUSY if a member still + * needs to be reissued (including a page re-dirtied during its write). It still + * waits until every issued request has finished, so no I/O belonging to the set + * remains in flight when it returns. + */ +platform_status +writeback_set_wait(writeback_set *set); + +/* Reissue every member not completely covered by its previous request. */ +platform_status +writeback_set_retry_incomplete(writeback_set *set); + +/* + * Make the completed writes durable. Must follow a successful + * writeback_set_wait(): the barrier covers writes that have completed, so + * calling it with members still in flight would not cover them. + * + * Today this is one cache-wide barrier, whose cost is O(1) in the size of the + * set and is shared with anything else that happens to have completed. The + * separate entry point leaves room for a narrower mechanism later without + * changing callers. + */ +platform_status +writeback_set_make_durable(writeback_set *set); + +/* + * Drop every member, keeping the allocation for reuse. Only sound once the + * members have been waited for: the requests are the only record that those + * writes are outstanding, so discarding them earlier loses the ability to tell + * when they land. + */ +void +writeback_set_reset(writeback_set *set); + +/* Number of members added so far. */ +uint64 +writeback_set_num_requests(const writeback_set *set); diff --git a/tests/config.c b/tests/config.c index 6123bb1c..068dc5c2 100644 --- a/tests/config.c +++ b/tests/config.c @@ -84,7 +84,9 @@ config_set_defaults(master_config *cfg) .filter_log_index_size = TEST_CONFIG_DEFAULT_FILTER_LOG_INDEX_SIZE, .use_log = FALSE, // 0 == follow cache_capacity; see master_config. - .checkpoint_log_size = 0, + .checkpoint_log_size = 0, + // 0 == twice memtable_capacity; see master_config. + .checkpoint_log_grace_bytes = 0, .num_normal_bg_threads = TEST_CONFIG_DEFAULT_NUM_NORMAL_BG_THREADS, .num_memtable_bg_threads = TEST_CONFIG_DEFAULT_NUM_MEMTABLE_BG_THREADS, .memtable_capacity = MiB_TO_B(TEST_CONFIG_DEFAULT_MEMTABLE_CAPACITY_MB), @@ -158,6 +160,10 @@ config_usage() platform_error_log("\t--checkpoint-log-size-gib\n"); platform_error_log("\t--checkpoint-log-size-mib\n"); platform_error_log("\t--checkpoint-log-size-bytes (0 => cache capacity)\n"); + platform_error_log("\t--checkpoint-log-grace-gib\n"); + platform_error_log("\t--checkpoint-log-grace-mib\n"); + platform_error_log( + "\t--checkpoint-log-grace-bytes (0 => twice memtable capacity)\n"); platform_error_log("\t--verbose-logging\n"); platform_error_log("\t--no-verbose-logging\n"); platform_error_log("\t--verbose-progress\n"); @@ -395,6 +401,16 @@ config_parse(master_config *cfg, const uint8 num_config, int argc, char *argv[]) "checkpoint-log-size-bytes", cfg, checkpoint_log_size) { } + config_set_mib("checkpoint-log-grace", cfg, checkpoint_log_grace_bytes) + { + } + config_set_gib("checkpoint-log-grace", cfg, checkpoint_log_grace_bytes) + { + } + config_set_uint64( + "checkpoint-log-grace-bytes", cfg, checkpoint_log_grace_bytes) + { + } config_set_mib("memtable-capacity", cfg, memtable_capacity) {} config_set_gib("memtable-capacity", cfg, memtable_capacity) {} config_set_uint64("rough-count-height", cfg, btree_rough_count_height) diff --git a/tests/config.h b/tests/config.h index 8fbdb8af..6c71afb5 100644 --- a/tests/config.h +++ b/tests/config.h @@ -76,13 +76,19 @@ typedef struct master_config { // log bool32 use_log; /* - * Take an automatic checkpoint once the live log has grown by this many + * Arm an automatic checkpoint once the live log has grown by this many * bytes. Zero means follow cache_capacity, matching the production default * of one cache's worth of log; it is resolved that way at use, so lowering * --cache-capacity also lowers this unless it is set explicitly. Set it * very large (UINT64_MAX) to effectively disable automatic checkpoints. */ uint64 checkpoint_log_size; + /* + * Additional log bytes allowed after the soft checkpoint trigger before a + * memtable rotation is forced. Zero follows the production default of + * twice memtable_capacity. + */ + uint64 checkpoint_log_grace_bytes; // task system uint64 num_normal_bg_threads; // Both bg_threads fields have to be non-zero diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 32b265e2..b06836b1 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -71,6 +71,13 @@ test_btree_process_noop(void *arg, uint64 generation) // really a no-op } +static bool32 +test_memtable_rotation_not_requested(void *arg) +{ + (void)arg; + return FALSE; +} + static platform_status test_memtable_generation_init(cache *cc, test_btree_config *cfg, @@ -109,6 +116,66 @@ test_memtable_generation_init(cache *cc, } } + /* + * A forced rotation must obey the same ring-capacity limit as a natural + * one. Leave every finalized generation outstanding (the process callback + * is a no-op), fill the ring, and verify that one more attempt reports BUSY + * without advancing the active generation or overwriting its output. + */ + for (uint64 generation = 0; generation + 1 < max_memtables; generation++) { + uint64 finalized_generation = UINT64_MAX; + rc = memtable_force_rotation(&mt_ctxt, &finalized_generation); + if (!SUCCESS(rc) || finalized_generation != generation) { + platform_error_log("forced memtable rotation for generation %lu " + "failed: %s (finalized %lu)\n", + generation, + platform_status_to_string(rc), + finalized_generation); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + } + uint64 generation_before_busy = mt_ctxt.generation; + uint64 finalized_generation = UINT64_MAX; + + /* + * A stale conditional request must be rejected before ring readiness is + * considered. In particular, a full ring must not turn a no-op request + * into STATUS_BUSY or advance the generation. + */ + rc = memtable_force_rotation_if(&mt_ctxt, + test_memtable_rotation_not_requested, + NULL, + &finalized_generation); + if (!SUCCESS(rc) || mt_ctxt.generation != generation_before_busy + || finalized_generation != UINT64_MAX) + { + platform_error_log("stale conditional rotation of a full memtable ring " + "returned %s; generation %lu -> %lu, output %lu\n", + platform_status_to_string(rc), + generation_before_busy, + mt_ctxt.generation, + finalized_generation); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + + rc = memtable_force_rotation(&mt_ctxt, &finalized_generation); + if (!STATUS_IS_EQ(rc, STATUS_BUSY) + || mt_ctxt.generation != generation_before_busy + || finalized_generation != UINT64_MAX) + { + platform_error_log("forced rotation of a full memtable ring returned %s; " + "generation %lu -> %lu, output %lu\n", + platform_status_to_string(rc), + generation_before_busy, + mt_ctxt.generation, + finalized_generation); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + rc = STATUS_OK; + deinit_fresh: memtable_context_deinit(&mt_ctxt); if (!SUCCESS(rc)) { diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index e774f90a..c36dfced 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -467,8 +467,8 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, * cache_get() calls, or those cache_get()s would trip the cache's debug * allocation check. */ - rc = mini_recover_allocations( - (cache *)&recovery_cc, meta_head, PAGE_TYPE_MISC); + rc = + mini_recover_references((cache *)&recovery_cc, meta_head, PAGE_TYPE_MISC); if (!SUCCESS(rc)) { platform_error_log("cache_test: mini_recover_allocations failed: %s\n", platform_status_to_string(rc)); @@ -481,7 +481,8 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, * refcount (which may include self-references beyond this) is explicitly * the job of a higher-level recovery walker, not this function. */ - if (allocator_get_refcount((allocator *)&recovery, meta_head) != AL_ONE_REF) + if (allocator_get_refcount((allocator *)&recovery, meta_head) + != AL_ONE_REF + 1) { platform_error_log( "cache_test: mini_recover_allocations did not recover the metadata " @@ -892,7 +893,8 @@ cache_test_hammer_thread(void *arg) memset(page->data, (uint8)i, ctxt->page_size); cache_unlock(ctxt->cc, page); cache_unclaim(ctxt->cc, page); - cache_page_writeback(ctxt->cc, page, FALSE, PAGE_TYPE_MISC); + // Other threads race us for these pages, so STATUS_BUSY is expected. + cache_writeback_page(ctxt->cc, page, PAGE_TYPE_MISC, NULL); } cache_unget(ctxt->cc, page); i++; diff --git a/tests/functional/io_apis_test.c b/tests/functional/io_apis_test.c index bc2064ae..3c96a8a7 100644 --- a/tests/functional/io_apis_test.c +++ b/tests/functional/io_apis_test.c @@ -33,6 +33,7 @@ * ---------------------------------------------------------------------------- */ #include +#include #include "platform_units.h" #include "platform_typed_alloc.h" @@ -131,6 +132,9 @@ test_async_reads_by_threads(io_test_fn_args *io_test_param, int nthreads, const char *whoami); +static void +test_readable_ranges(platform_heap_id hid, master_config *master_cfg); + static void load_thread_params(io_test_fn_args *io_test_param, io_test_fn_args *thread_params, @@ -204,6 +208,8 @@ splinter_io_apis_test(int argc, char *argv[]) Verbose_progress = master_cfg.verbose_progress; + test_readable_ranges(hid, &master_cfg); + // Ensure that the default max async q-depth configured for // master-cfg is sufficiently big enough for this test case. platform_assert(NUM_PAGES_RW_ASYNC_PER_THREAD @@ -411,6 +417,122 @@ splinter_io_apis_test(int argc, char *argv[]) return (SUCCESS(rc) ? 0 : -1); } +/* + * Exercise the logical-size cache on an isolated, deliberately partial-page + * regular file. In particular, both write paths must invalidate a size that + * was cached before they extended the file. + */ +static void +test_readable_ranges(platform_heap_id hid, master_config *master_cfg) +{ + char filename[MAX_STRING_LENGTH]; + int n = snprintf(filename, + sizeof(filename), + "/tmp/splinterdb-io-range-%d.db", + platform_get_os_pid()); + platform_assert(n > 0 && n < sizeof(filename)); + + int fd = open(filename, O_RDWR | O_CREAT | O_TRUNC, master_cfg->io_perms); + platform_assert(fd >= 0, "open(%s) failed: %s", filename, strerror(errno)); + + uint64 page_size = master_cfg->page_size; + uint64 extent_size = master_cfg->extent_size; + uint64 initial_size = extent_size + page_size / 2; + int sys_rc = ftruncate(fd, initial_size); + platform_assert( + sys_rc == 0, "ftruncate(%s) failed: %s", filename, strerror(errno)); + sys_rc = close(fd); + platform_assert( + sys_rc == 0, "close(%s) failed: %s", filename, strerror(errno)); + + io_config cfg; + io_config_init(&cfg, + page_size, + extent_size, + master_cfg->io_flags, + master_cfg->io_perms, + master_cfg->io_async_queue_depth, + filename); + io_handle *io = io_handle_create(&cfg, hid); + platform_assert(io != NULL); + + bool32 readable = FALSE; + platform_status rc = io_range_is_readable(io, 0, initial_size, &readable); + platform_assert_status_ok(rc); + platform_assert(readable); + + uint64 partial_page = initial_size - page_size / 2; + rc = io_range_is_readable(io, partial_page, page_size / 2, &readable); + platform_assert_status_ok(rc); + platform_assert(readable); + rc = io_range_is_readable(io, partial_page, page_size, &readable); + platform_assert_status_ok(rc); + platform_assert(!readable); + + rc = io_range_is_readable(io, initial_size, 1, &readable); + platform_assert_status_ok(rc); + platform_assert(!readable); + rc = io_range_is_readable(io, initial_size, 0, &readable); + platform_assert_status_ok(rc); + platform_assert(readable); + rc = io_range_is_readable(io, initial_size + 1, 0, &readable); + platform_assert_status_ok(rc); + platform_assert(!readable); + + readable = TRUE; + rc = io_range_is_readable(io, UINT64_MAX - 1, 3, &readable); + platform_assert(STATUS_IS_EQ(rc, STATUS_BAD_PARAM)); + platform_assert(!readable); + + char *buf = TYPED_ARRAY_ZALLOC(hid, buf, page_size); + platform_assert(buf != NULL); + + /* A read that crosses EOF is a hard error, even when it starts in-range. */ + rc = io_read(io, buf, page_size, partial_page); + platform_assert(!SUCCESS(rc)); + + uint64 sync_addr = ROUNDUP(initial_size, page_size); + rc = io_range_is_readable(io, sync_addr, page_size, &readable); + platform_assert_status_ok(rc); + platform_assert(!readable); // primes the pre-extension cached size + rc = io_write(io, buf, page_size, sync_addr); + platform_assert_status_ok(rc); + rc = io_range_is_readable(io, sync_addr, page_size, &readable); + platform_assert_status_ok(rc); + platform_assert(readable); + rc = io_range_is_readable( + io, initial_size, sync_addr + page_size - initial_size, &readable); + platform_assert_status_ok(rc); + platform_assert(readable); // includes the sparse gap after the old EOF + + uint64 async_addr = sync_addr + page_size; + rc = io_range_is_readable(io, async_addr, page_size, &readable); + platform_assert_status_ok(rc); + platform_assert(!readable); // primes the pre-extension cached size again + + io_async_state_buffer state; + rc = + io_async_state_init(state, io, io_async_pwritev, async_addr, NULL, NULL); + platform_assert_status_ok(rc); + rc = io_async_state_append_page(state, buf); + platform_assert_status_ok(rc); + io_async_run(state); + io_wait_all(io); + rc = io_async_state_get_result(state); + platform_assert_status_ok(rc); + io_async_state_deinit(state); + + rc = io_range_is_readable(io, async_addr, page_size, &readable); + platform_assert_status_ok(rc); + platform_assert(readable); + + platform_free(hid, buf); + io_handle_destroy(io); + sys_rc = unlink(filename); + platform_assert( + sys_rc == 0, "unlink(%s) failed: %s", filename, strerror(errno)); +} + /* * ----------------------------------------------------------------------------- * test_sync_writes() - Write out a swath of disk using page-sized sync-write diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index 2cdcb39d..f16aeb2e 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -7,6 +7,7 @@ * This file contains tests for Alex's log */ #include "platform_time.h" +#include "platform_sleep.h" #include "log.h" #include "shard_log.h" #include "platform_io.h" @@ -48,8 +49,7 @@ test_log_crash(clockcache *cc, DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); platform_assert(cc != NULL); - logh = shard_log_create((cache *)cc, cfg, hid); - platform_assert(logh != NULL); + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &logh)); // The identity is fixed at creation; capture it before writing/sealing. segment = log_get_head(logh); @@ -88,8 +88,9 @@ test_log_crash(clockcache *cc, platform_assert(log_rc == 0); } - rc = log_seal(logh); // frees logh; identity captured above + rc = log_seal(logh); // identity captured above platform_assert_status_ok(rc); + log_deinit(logh); rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); rc = cache_durable_barrier((cache *)cc); @@ -102,8 +103,10 @@ test_log_crash(clockcache *cc, platform_assert_status_ok(rc); } - itor = shard_log_iterator_create((cache *)cc, cfg, hid, segment); - platform_assert(itor != NULL); + platform_assert_status_ok( + shard_log_iterator_create((cache *)cc, cfg, hid, segment, 0, &itor)); + // The stream was sealed, so replay must be able to see that it is whole. + platform_assert(log_iterator_stream_complete(itor)); for (i = 0; i < num_entries && log_iterator_can_next(itor); i++) { key skey = @@ -140,7 +143,7 @@ test_log_crash(clockcache *cc, merge_accumulator_deinit(&msg); log_iterator_deinit(itor); - log_dec_ref((cache *)cc, &segment); + shard_log_dec_ref((cache *)cc, &segment); return 0; } @@ -193,8 +196,9 @@ test_log_verify_segment(cache *cc, platform_assert(segment->addr != 0); platform_assert(segment->meta_addr != 0); - itor = shard_log_iterator_create(cc, cfg, hid, *segment); - platform_assert(itor != NULL); + platform_assert_status_ok( + shard_log_iterator_create(cc, cfg, hid, *segment, 0, &itor)); + platform_assert(log_iterator_stream_complete(itor)); merge_accumulator_init(&msg, hid); for (uint64 i = 0; i < num_entries; i++) { @@ -228,6 +232,1126 @@ test_log_verify_segment(cache *cc, log_iterator_deinit(itor); } +typedef struct test_log_reserved_writer_params { + log_handle *log; + platform_thread thread; + test_message_generator *gen; + platform_heap_id hid; + uint64 key_size; + uint64 entry; + volatile bool32 reserved; + volatile bool32 release; + int append_rc; +} test_log_reserved_writer_params; + +static void +test_log_reserved_writer(void *arg) +{ + test_log_reserved_writer_params *params = arg; + merge_accumulator msg; + DECLARE_AUTO_KEY_BUFFER(keybuffer, params->hid); + + merge_accumulator_init(&msg, params->hid); + key skey = test_key(&keybuffer, + TEST_RANDOM, + params->entry, + 0, + 0, + 1 + (params->entry % params->key_size), + 0); + generate_test_message(params->gen, params->entry, &msg); + + log_write_token reserved; + log_write_reserve(params->log, &reserved); + __atomic_store_n(¶ms->reserved, TRUE, __ATOMIC_RELEASE); + while (!__atomic_load_n(¶ms->release, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + + params->append_rc = log_write_reserved( + &reserved, skey, merge_accumulator_to_message(&msg), params->entry, 0); + platform_assert(reserved.log == NULL); + platform_assert(reserved.internal == NULL); + merge_accumulator_deinit(&msg); +} + +static bool32 +test_log_wait_for_flag(volatile bool32 *flag) +{ + uint64 start = platform_get_timestamp(); + while (!__atomic_load_n(flag, __ATOMIC_ACQUIRE) + && platform_timestamp_elapsed(start) < SEC_TO_NSEC(10)) + { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return __atomic_load_n(flag, __ATOMIC_ACQUIRE); +} + +/* + * log_make_durable() mid-stream must produce a stream of several groups that + * still replays as one sequence. + * + * This is the only coverage of the reader's multi-group path: that group ids + * are dense, that a run of them is accepted in order, and that the records of + * every closed group survive. Sealing alone leaves a single group, so nothing + * else reaches it. + */ +static int +test_log_multiple_groups(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + const uint64 num_groups = 4; + const uint64 per_group = 32; + log_head segment; + + log_handle *log; + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); + platform_assert(log_is_empty(log)); + log_durable_ticket empty_ticket; + platform_assert_status_ok(log_make_durable_begin(log, &empty_ticket)); + platform_assert(empty_ticket == 0); + platform_assert_status_ok(log_make_durable_wait(log, empty_ticket)); + platform_assert(log_is_empty(log)); + segment = log_get_head(log); + + for (uint64 g = 0; g < num_groups; g++) { + test_log_write_range(log, gen, hid, key_size, g * per_group, per_group); + platform_assert(!log_is_empty(log)); + // Ends this group and starts the next; the stream stays open. + log_durable_ticket cut_ticket; + platform_assert_status_ok(log_make_durable_begin(log, &cut_ticket)); + platform_assert_status_ok(log_make_durable_wait(log, cut_ticket)); + + /* The empty successor's frontier is exactly the group just closed. */ + platform_assert_status_ok(log_make_durable_begin(log, &empty_ticket)); + platform_assert(empty_ticket == cut_ticket); + platform_assert_status_ok(log_make_durable_wait(log, empty_ticket)); + platform_assert(!log_is_empty(log)); + } + + platform_assert_status_ok(log_seal(log)); + platform_assert(!log_is_empty(log)); + log_deinit(log); + + platform_status rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + // Re-read from a cold cache so only persisted pages are consulted. + clockcache_deinit(cc); + rc = clockcache_init( + cc, cache_cfg, io, al, "multi-group", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + + // Every record of every group must come back, as one sequence. + test_log_verify_segment((cache *)cc, + cfg, + &segment, + gen, + hid, + key_size, + 0, + num_groups * per_group); + + shard_log_dec_ref((cache *)cc, &segment); + return 0; +} + +/* + * Split-phase durability must permit several cuts to be staged before any + * caller waits. Waiting for the newest ticket first makes all earlier groups + * durable with one contiguous barrier; the older tickets then merely consume + * their pins. Enough records are used to force ordinary data-page graduation + * in later groups before their explicit close, exercising physical ordering as + * well as the partial-page close path. + */ +static int +test_log_pipelined_groups(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + const uint64 per_group = 256; + log_handle *log; + log_head segment; + log_durable_ticket first_ticket; + log_durable_ticket second_ticket; + + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); + segment = log_get_head(log); + + /* + * Leave the last record reserved across the cut. It must remain attached to + * the group selected by reserve, and the cut must not graduate that group + * until log_write_reserved() consumes the reservation. + */ + test_log_write_range(log, gen, hid, key_size, 0, per_group - 1); + test_log_reserved_writer_params writer = { + .log = log, + .gen = gen, + .hid = hid, + .key_size = key_size, + .entry = per_group - 1, + .append_rc = -1, + }; + platform_assert_status_ok(platform_thread_create( + &writer.thread, FALSE, test_log_reserved_writer, &writer, hid)); + bool32 writer_reserved = test_log_wait_for_flag(&writer.reserved); + if (!writer_reserved) { + __atomic_store_n(&writer.release, TRUE, __ATOMIC_RELEASE); + platform_thread_join(&writer.thread); + } + platform_assert(writer_reserved, + "reserved writer did not publish its reservation"); + + platform_assert_status_ok(log_make_durable_begin(log, &first_ticket)); + platform_assert(first_ticket != 0); + __atomic_store_n(&writer.release, TRUE, __ATOMIC_RELEASE); + platform_thread_join(&writer.thread); + platform_assert(writer.append_rc == 0); + + test_log_write_range(log, gen, hid, key_size, per_group, per_group); + platform_assert_status_ok(log_make_durable_begin(log, &second_ticket)); + platform_assert(second_ticket > first_ticket); + + test_log_write_range(log, gen, hid, key_size, 2 * per_group, per_group); + + /* The newer waiter may drive and cover the whole contiguous prefix. */ + platform_assert_status_ok(log_make_durable_wait(log, second_ticket)); + platform_assert_status_ok(log_make_durable_wait(log, first_ticket)); + platform_assert_status_ok(log_seal(log)); + log_deinit(log); + + platform_status rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init( + cc, cache_cfg, io, al, "pipelined-groups", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + + test_log_verify_segment( + (cache *)cc, cfg, &segment, gen, hid, key_size, 0, 3 * per_group); + shard_log_dec_ref((cache *)cc, &segment); + return 0; +} + +typedef struct test_log_concurrent_begin_params { + log_handle *log; + platform_thread thread; + volatile bool32 *start; + volatile bool32 *release; + volatile bool32 began; + platform_status begin_rc; + platform_status wait_rc; + log_durable_ticket ticket; +} test_log_concurrent_begin_params; + +static void +test_log_concurrent_begin(void *arg) +{ + test_log_concurrent_begin_params *params = arg; + while (!__atomic_load_n(params->start, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + + params->begin_rc = log_make_durable_begin(params->log, ¶ms->ticket); + __atomic_store_n(¶ms->began, TRUE, __ATOMIC_RELEASE); + while (!__atomic_load_n(params->release, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + + params->wait_rc = params->begin_rc; + if (SUCCESS(params->begin_rc)) { + params->wait_rc = log_make_durable_wait(params->log, params->ticket); + } +} + +static bool32 +test_log_wait_for_begins(test_log_concurrent_begin_params *params, + uint64 num_threads) +{ + uint64 start = platform_get_timestamp(); + while (platform_timestamp_elapsed(start) < SEC_TO_NSEC(10)) { + bool32 all_began = TRUE; + for (uint64 i = 0; i < num_threads; i++) { + all_began &= __atomic_load_n(¶ms[i].began, __ATOMIC_ACQUIRE); + } + if (all_began) { + return TRUE; + } + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return FALSE; +} + +static log_durable_ticket +test_log_concurrent_begin_round(log_handle *log, + platform_heap_id hid, + uint64 num_threads) +{ + platform_assert(num_threads <= MAX_THREADS); + test_log_concurrent_begin_params params[MAX_THREADS] = {0}; + volatile bool32 start = FALSE; + volatile bool32 release = FALSE; + + for (uint64 i = 0; i < num_threads; i++) { + params[i] = (test_log_concurrent_begin_params){ + .log = log, + .start = &start, + .release = &release, + .begin_rc = STATUS_INVALID_STATE, + .wait_rc = STATUS_INVALID_STATE, + }; + platform_assert_status_ok(platform_thread_create( + ¶ms[i].thread, FALSE, test_log_concurrent_begin, ¶ms[i], hid)); + } + + __atomic_store_n(&start, TRUE, __ATOMIC_RELEASE); + bool32 all_began = test_log_wait_for_begins(params, num_threads); + bool32 coalesced = all_began; + for (uint64 i = 1; i < num_threads && coalesced; i++) { + coalesced = params[i].ticket == params[0].ticket; + } + + __atomic_store_n(&release, TRUE, __ATOMIC_RELEASE); + for (uint64 i = 0; i < num_threads; i++) { + platform_thread_join(¶ms[i].thread); + } + + platform_assert(all_began, "concurrent durability begin timed out"); + platform_assert(coalesced, "concurrent durability begins did not coalesce"); + platform_assert(params[0].ticket != 0); + for (uint64 i = 0; i < num_threads; i++) { + platform_assert_status_ok(params[i].begin_rc); + platform_assert_status_ok(params[i].wait_rc); + } + return params[0].ticket; +} + +/* + * Concurrent begin calls racing to cut one used group must all cover the same + * frontier. Repeating the race after appending to the successor verifies that + * the versioned installation claim hands off to the next group rather than + * looking like a stale claim left behind by the previous installer. + */ +static int +test_log_concurrent_begin_handoff(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + const uint64 num_cutters = 8; + log_handle *log; + + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); + log_head segment = log_get_head(log); + + shard_log *slog = (shard_log *)log; + shard_log_group *initial = + __atomic_load_n(&slog->accepting.group, __ATOMIC_ACQUIRE); + platform_assert(initial != NULL); + platform_assert(initial->id == SHARD_LOG_FIRST_GROUP_ID); + platform_assert(__atomic_load_n(&slog->accepting.id, __ATOMIC_ACQUIRE) + == initial->id); + platform_assert(__atomic_load_n(&slog->install.state, __ATOMIC_ACQUIRE) + == initial->id); + + test_log_write_range(log, gen, hid, key_size, 0, 1); + log_durable_ticket first = + test_log_concurrent_begin_round(log, hid, num_cutters); + platform_assert(first == SHARD_LOG_FIRST_GROUP_ID); + shard_log_group *successor = + __atomic_load_n(&slog->accepting.group, __ATOMIC_ACQUIRE); + platform_assert(successor != NULL); + platform_assert(successor->id == first + 1); + platform_assert(__atomic_load_n(&slog->accepting.id, __ATOMIC_ACQUIRE) + == successor->id); + + test_log_write_range(log, gen, hid, key_size, 1, 1); + log_durable_ticket second = + test_log_concurrent_begin_round(log, hid, num_cutters); + platform_assert(second == first + 1); + + platform_assert_status_ok(log_seal(log)); + log_deinit(log); + + platform_status rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init(cc, + cache_cfg, + io, + al, + "concurrent-begin-handoff", + hid, + platform_get_module_id()); + platform_assert_status_ok(rc); + test_log_verify_segment( + (cache *)cc, cfg, &segment, gen, hid, key_size, 0, 2); + shard_log_dec_ref((cache *)cc, &segment); + return 0; +} + +typedef struct test_log_begin_actor { + log_handle *log; + platform_thread thread; + volatile bool32 entered; + volatile bool32 began; + volatile bool32 done; + platform_status begin_rc; + platform_status wait_rc; + log_durable_ticket ticket; +} test_log_begin_actor; + +static void +test_log_begin_actor_run(void *arg) +{ + test_log_begin_actor *actor = arg; + + __atomic_store_n(&actor->entered, TRUE, __ATOMIC_RELEASE); + actor->begin_rc = log_make_durable_begin(actor->log, &actor->ticket); + __atomic_store_n(&actor->began, TRUE, __ATOMIC_RELEASE); + actor->wait_rc = actor->begin_rc; + if (SUCCESS(actor->begin_rc)) { + actor->wait_rc = log_make_durable_wait(actor->log, actor->ticket); + } + __atomic_store_n(&actor->done, TRUE, __ATOMIC_RELEASE); +} + +typedef struct test_log_seal_actor { + log_handle *log; + platform_thread thread; + volatile bool32 entered; + volatile bool32 done; + platform_status rc; +} test_log_seal_actor; + +static void +test_log_seal_actor_run(void *arg) +{ + test_log_seal_actor *actor = arg; + + __atomic_store_n(&actor->entered, TRUE, __ATOMIC_RELEASE); + actor->rc = log_seal(actor->log); + __atomic_store_n(&actor->done, TRUE, __ATOMIC_RELEASE); +} + +static bool32 +test_log_wait_for_install_state(shard_log *log, + uint64 expected, + bool32 accepting_is_null) +{ + uint64 start = platform_get_timestamp(); + while (platform_timestamp_elapsed(start) < SEC_TO_NSEC(10)) { + uint64 state = __atomic_load_n(&log->install.state, __ATOMIC_ACQUIRE); + shard_log_group *accepting = + __atomic_load_n(&log->accepting.group, __ATOMIC_ACQUIRE); + if (state == expected && ((accepting == NULL) == accepting_is_null)) { + return TRUE; + } + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return FALSE; +} + +static void +test_log_assert_begin_actor(const test_log_begin_actor *actor) +{ + platform_assert(__atomic_load_n(&actor->began, __ATOMIC_ACQUIRE)); + platform_assert(__atomic_load_n(&actor->done, __ATOMIC_ACQUIRE)); + platform_assert_status_ok(actor->begin_rc); + platform_assert(actor->ticket != 0); + platform_assert_status_ok(actor->wait_rc); +} + +static void +test_log_finish_claim_race(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size, + log_handle *log, + const log_head *segment, + char *cache_name, + uint64 first_entry, + uint64 num_entries) +{ + /* A second seal must remain a successful no-op after either race. */ + platform_assert_status_ok(log_seal(log)); + log_deinit(log); + + platform_assert_status_ok(cache_writeback_dirty((cache *)cc)); + platform_assert_status_ok(cache_durable_barrier((cache *)cc)); + + clockcache_deinit(cc); + platform_assert_status_ok(clockcache_init( + cc, cache_cfg, io, al, cache_name, hid, platform_get_module_id())); + test_log_verify_segment( + (cache *)cc, cfg, segment, gen, hid, key_size, first_entry, num_entries); + shard_log_dec_ref((cache *)cc, segment); +} + +/* + * Stop an installation immediately after its atomic claim, before the cut is + * published. A concurrent begin must return the claimed group's ticket without + * chasing a successor, while its wait must remain blocked until some caller + * finishes that cut. + */ +static int +test_log_begin_loses_install_claim(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + log_handle *log; + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); + log_head segment = log_get_head(log); + test_log_write_range(log, gen, hid, key_size, 0, 1); + + shard_log *slog = (shard_log *)log; + shard_log_group *current = + __atomic_load_n(&slog->accepting.group, __ATOMIC_ACQUIRE); + platform_assert(current != NULL); + log_durable_ticket target = current->id; + + /* Synthesize a winning caller paused between its claim and publication. */ + uint64 expected_state = target; + bool32 claimed = __atomic_compare_exchange_n(&slog->install.state, + &expected_state, + target + 1, + FALSE, + __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST); + platform_assert(claimed); + + test_log_begin_actor loser = { + .log = log, + .begin_rc = STATUS_INVALID_STATE, + .wait_rc = STATUS_INVALID_STATE, + }; + platform_assert_status_ok(platform_thread_create( + &loser.thread, FALSE, test_log_begin_actor_run, &loser, hid)); + + bool32 loser_began = test_log_wait_for_flag(&loser.began); + if (!loser_began) { + /* Let a broken implementation escape its retry loop before asserting. */ + __atomic_store_n(&slog->install.state, target, __ATOMIC_SEQ_CST); + platform_thread_join(&loser.thread); + } + platform_assert(loser_began, + "begin retried after losing the installation claim"); + platform_assert_status_ok(loser.begin_rc); + platform_assert(loser.ticket == target, + "begin advanced from ticket %lu to %lu", + target, + loser.ticket); + platform_assert(!__atomic_load_n(&loser.done, __ATOMIC_ACQUIRE), + "ticket wait graduated an OPEN group"); + + /* + * Relinquish the synthetic claim, then use the production path to publish + * the cut. The already-returned ticket waiter must now be able to finish. + */ + expected_state = target + 1; + bool32 released = __atomic_compare_exchange_n(&slog->install.state, + &expected_state, + target, + FALSE, + __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST); + platform_assert(released); + + log_durable_ticket publisher_ticket; + platform_assert_status_ok(log_make_durable_begin(log, &publisher_ticket)); + platform_assert(publisher_ticket == target); + platform_assert_status_ok(log_make_durable_wait(log, publisher_ticket)); + + platform_thread_join(&loser.thread); + test_log_assert_begin_actor(&loser); + platform_assert(loser.ticket == target); + + test_log_finish_claim_race(cc, + cache_cfg, + io, + al, + cfg, + hid, + gen, + key_size, + log, + &segment, + "begin-loses-install-claim", + 0, + 1); + return 0; +} + +#if SPLINTER_DEBUG +typedef struct test_log_publication_hook { + uint64 group_id; + volatile bool32 successor_published; + volatile bool32 handoff_waited; + volatile bool32 release_successor; +} test_log_publication_hook; + +static void +test_log_publication_hook_run(void *arg, + shard_log_test_hook_event event, + uint64 group_id) +{ + test_log_publication_hook *hook = arg; + if (group_id != hook->group_id) { + return; + } + + if (event == SHARD_LOG_TEST_SUCCESSOR_POINTER_PUBLISHED) { + __atomic_store_n(&hook->successor_published, TRUE, __ATOMIC_RELEASE); + while (!__atomic_load_n(&hook->release_successor, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + } else { + platform_assert(event == SHARD_LOG_TEST_ACCEPTING_HANDOFF_WAIT); + __atomic_store_n(&hook->handoff_waited, TRUE, __ATOMIC_RELEASE); + } +} + +static bool32 +test_log_wait_for_handoff_or_done(const test_log_publication_hook *hook, + const volatile bool32 *escape) +{ + uint64 start = platform_get_timestamp(); + while (platform_timestamp_elapsed(start) < SEC_TO_NSEC(10)) { + if (__atomic_load_n(&hook->handoff_waited, __ATOMIC_ACQUIRE)) { + return TRUE; + } + if (__atomic_load_n(escape, __ATOMIC_ACQUIRE)) { + return FALSE; + } + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return FALSE; +} + +/* + * Pause one cutter after publishing its successor pointer but before its id. + * A cutter of that now-used successor may win the following claim, but must + * wait for the first cutter's id handoff before changing or publishing it. + */ +static int +test_log_nested_cutter_publication_handoff(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + log_handle *log; + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); + log_head segment = log_get_head(log); + test_log_write_range(log, gen, hid, key_size, 0, 1); + + shard_log *slog = (shard_log *)log; + shard_log_group *initial = + __atomic_load_n(&slog->accepting.group, __ATOMIC_ACQUIRE); + platform_assert(initial != NULL); + uint64 initial_id = initial->id; + test_log_publication_hook hook = { + .group_id = initial_id + 1, + }; + slog->test_hook = test_log_publication_hook_run; + slog->test_hook_arg = &hook; + + test_log_begin_actor first = { + .log = log, + .begin_rc = STATUS_INVALID_STATE, + .wait_rc = STATUS_INVALID_STATE, + }; + platform_assert_status_ok(platform_thread_create( + &first.thread, FALSE, test_log_begin_actor_run, &first, hid)); + platform_assert(test_log_wait_for_flag(&hook.successor_published), + "first cutter did not publish its successor pointer"); + + shard_log_group *successor = + __atomic_load_n(&slog->accepting.group, __ATOMIC_ACQUIRE); + platform_assert(successor != NULL); + platform_assert(successor->id == hook.group_id); + uint64 successor_id = successor->id; + platform_assert(__atomic_load_n(&slog->accepting.id, __ATOMIC_ACQUIRE) + == initial_id); + test_log_write_range(log, gen, hid, key_size, 1, 1); + + test_log_begin_actor second = { + .log = log, + .begin_rc = STATUS_INVALID_STATE, + .wait_rc = STATUS_INVALID_STATE, + }; + platform_assert_status_ok(platform_thread_create( + &second.thread, FALSE, test_log_begin_actor_run, &second, hid)); + platform_assert( + test_log_wait_for_handoff_or_done(&hook, &second.began), + "nested cutter did not wait for the predecessor id publication"); + platform_assert(__atomic_load_n(&slog->install.state, __ATOMIC_ACQUIRE) + == successor_id + 1); + platform_assert(__atomic_load_n(&slog->accepting.group, __ATOMIC_ACQUIRE) + == successor); + platform_assert(__atomic_load_n(&slog->accepting.id, __ATOMIC_ACQUIRE) + == initial_id); + + __atomic_store_n(&hook.release_successor, TRUE, __ATOMIC_RELEASE); + platform_thread_join(&first.thread); + platform_thread_join(&second.thread); + test_log_assert_begin_actor(&first); + test_log_assert_begin_actor(&second); + platform_assert(first.ticket == initial_id); + platform_assert(second.ticket == successor_id); + slog->test_hook = NULL; + slog->test_hook_arg = NULL; + + test_log_finish_claim_race(cc, + cache_cfg, + io, + al, + cfg, + hid, + gen, + key_size, + log, + &segment, + "nested-cutter-publication-handoff", + 0, + 2); + return 0; +} + +/* + * Pause a cutter after publishing its successor pointer but before its id. + * Seal may win the terminal claim for that successor, but must wait for the + * predecessor's id handoff before closing it and publishing NULL. + */ +static int +test_log_begin_claim_precedes_seal(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + log_handle *log; + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); + log_head segment = log_get_head(log); + test_log_write_range(log, gen, hid, key_size, 0, 1); + + shard_log *slog = (shard_log *)log; + shard_log_group *initial = + __atomic_load_n(&slog->accepting.group, __ATOMIC_ACQUIRE); + platform_assert(initial != NULL); + uint64 initial_id = initial->id; + test_log_publication_hook hook = { + .group_id = initial_id + 1, + }; + slog->test_hook = test_log_publication_hook_run; + slog->test_hook_arg = &hook; + + test_log_begin_actor begin = { + .log = log, + .begin_rc = STATUS_INVALID_STATE, + .wait_rc = STATUS_INVALID_STATE, + }; + platform_assert_status_ok(platform_thread_create( + &begin.thread, FALSE, test_log_begin_actor_run, &begin, hid)); + platform_assert(test_log_wait_for_flag(&hook.successor_published), + "begin did not publish its successor pointer"); + shard_log_group *successor = + __atomic_load_n(&slog->accepting.group, __ATOMIC_ACQUIRE); + platform_assert(successor != NULL); + platform_assert(successor->id == hook.group_id); + uint64 successor_id = successor->id; + platform_assert(__atomic_load_n(&slog->accepting.id, __ATOMIC_ACQUIRE) + == initial_id); + + test_log_seal_actor seal = { + .log = log, + .rc = STATUS_INVALID_STATE, + }; + platform_assert_status_ok(platform_thread_create( + &seal.thread, FALSE, test_log_seal_actor_run, &seal, hid)); + platform_assert(test_log_wait_for_handoff_or_done(&hook, &seal.done), + "seal did not wait for the predecessor id publication"); + platform_assert(__atomic_load_n(&slog->install.state, __ATOMIC_ACQUIRE) + == (SHARD_LOG_INSTALL_TERMINAL_BIT | successor_id)); + platform_assert(__atomic_load_n(&slog->accepting.group, __ATOMIC_ACQUIRE) + == successor); + platform_assert(__atomic_load_n(&slog->accepting.id, __ATOMIC_ACQUIRE) + == initial_id); + + __atomic_store_n(&hook.release_successor, TRUE, __ATOMIC_RELEASE); + platform_thread_join(&begin.thread); + platform_thread_join(&seal.thread); + test_log_assert_begin_actor(&begin); + platform_assert(__atomic_load_n(&seal.done, __ATOMIC_ACQUIRE)); + platform_assert_status_ok(seal.rc); + slog->test_hook = NULL; + slog->test_hook_arg = NULL; + + test_log_finish_claim_race(cc, + cache_cfg, + io, + al, + cfg, + hid, + gen, + key_size, + log, + &segment, + "begin-claim-before-seal", + 0, + 1); + return 0; +} +#endif + +/* + * Hold a real write reservation so seal can publish its terminal claim and + * remove the accepting group but cannot finish graduating it. A concurrent + * begin must return the seal ticket while both its wait and seal itself remain + * blocked on that reservation. Consuming the reservation then releases both. + */ +static int +test_log_seal_claim_precedes_begin(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + log_handle *log; + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); + log_head segment = log_get_head(log); + test_log_write_range(log, gen, hid, key_size, 1, 1); + + test_log_reserved_writer_params writer = { + .log = log, + .gen = gen, + .hid = hid, + .key_size = key_size, + .entry = 2, + .append_rc = -1, + }; + platform_assert_status_ok(platform_thread_create( + &writer.thread, FALSE, test_log_reserved_writer, &writer, hid)); + bool32 writer_reserved = test_log_wait_for_flag(&writer.reserved); + if (!writer_reserved) { + __atomic_store_n(&writer.release, TRUE, __ATOMIC_RELEASE); + platform_thread_join(&writer.thread); + } + platform_assert(writer_reserved, + "writer did not publish its reservation before seal"); + + shard_log *slog = (shard_log *)log; + shard_log_group *current = + __atomic_load_n(&slog->accepting.group, __ATOMIC_ACQUIRE); + platform_assert(current != NULL); + uint64 terminal_state = SHARD_LOG_INSTALL_TERMINAL_BIT | current->id; + + test_log_seal_actor seal = { + .log = log, + .rc = STATUS_INVALID_STATE, + }; + platform_assert_status_ok(platform_thread_create( + &seal.thread, FALSE, test_log_seal_actor_run, &seal, hid)); + bool32 seal_claimed = + test_log_wait_for_install_state(slog, terminal_state, TRUE); + if (!seal_claimed) { + __atomic_store_n(&writer.release, TRUE, __ATOMIC_RELEASE); + platform_thread_join(&writer.thread); + platform_thread_join(&seal.thread); + } + platform_assert(seal_claimed, + "seal did not publish its terminal installation claim"); + platform_assert(!__atomic_load_n(&seal.done, __ATOMIC_ACQUIRE), + "seal ignored an outstanding write reservation"); + + test_log_begin_actor begin = { + .log = log, + .begin_rc = STATUS_INVALID_STATE, + .wait_rc = STATUS_INVALID_STATE, + }; + platform_assert_status_ok(platform_thread_create( + &begin.thread, FALSE, test_log_begin_actor_run, &begin, hid)); + bool32 begin_returned = test_log_wait_for_flag(&begin.began); + platform_assert(begin_returned, + "begin did not return the in-progress seal ticket"); + platform_assert_status_ok(begin.begin_rc); + platform_assert(begin.ticket != 0); + platform_assert(!__atomic_load_n(&begin.done, __ATOMIC_ACQUIRE), + "begin wait ignored an outstanding write reservation"); + + __atomic_store_n(&writer.release, TRUE, __ATOMIC_RELEASE); + platform_thread_join(&writer.thread); + platform_thread_join(&begin.thread); + platform_thread_join(&seal.thread); + platform_assert(writer.append_rc == 0); + test_log_assert_begin_actor(&begin); + platform_assert(__atomic_load_n(&seal.done, __ATOMIC_ACQUIRE)); + platform_assert_status_ok(seal.rc); + + test_log_finish_claim_race(cc, + cache_cfg, + io, + al, + cfg, + hid, + gen, + key_size, + log, + &segment, + "seal-claim-before-begin", + 1, + 2); + return 0; +} + +/* + * A log append happens after its corresponding memtable update is visible, so + * any append failure permanently invalidates that durability group. Verify + * that the first failure is retained by both wait and seal, that a clean + * predecessor can still become durable, and that neither records already + * staged in the poisoned group nor records accepted into a later group become + * replayable. + */ +static int +test_log_append_failure_poison(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + cache *cacheh = (cache *)cc; + log_handle *log; + log_iterator *itor; + + platform_assert_status_ok(shard_log_create(cacheh, cfg, hid, &log)); + log_head segment = log_get_head(log); + + /* Leave the clean predecessor unwaited so the poisoned wait must drive it. + */ + test_log_write_range(log, gen, hid, key_size, 0, 1); + log_durable_ticket clean_ticket; + platform_assert_status_ok(log_make_durable_begin(log, &clean_ticket)); + + /* + * Force ordinary data pages out of the next group before poisoning it. The + * missing terminator must make recovery discard every one of those pages. + */ + test_log_write_range(log, gen, hid, key_size, 1, 256); + + blob invalid_blob = { + .length = 0, + .checksum = {0}, + .format = BLOB_FORMAT + 1, + }; + message invalid_msg = + message_create(MESSAGE_TYPE_INSERT, + cacheh, + slice_create(sizeof(invalid_blob), &invalid_blob)); + DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); + key invalid_key = test_key(&keybuffer, TEST_RANDOM, 257, 0, 0, key_size, 0); + int append_rc = log_write(log, invalid_key, invalid_msg, 257, 0); + platform_assert(append_rc == STATUS_INVALID_STATE.r); + + log_durable_ticket poisoned_ticket; + platform_assert_status_ok(log_make_durable_begin(log, &poisoned_ticket)); + platform_assert(poisoned_ticket > clean_ticket); + + /* A raw later append may stage, but it must never cross the poison on disk. + */ + test_log_write_range(log, gen, hid, key_size, 258, 1); + + platform_status rc = log_make_durable_wait(log, poisoned_ticket); + platform_assert(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); + rc = log_make_durable_wait(log, clean_ticket); + platform_assert_status_ok(rc); + + rc = log_seal(log); + platform_assert(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); + log_deinit(log); + + rc = cache_writeback_dirty(cacheh); + platform_assert_status_ok(rc); + rc = cache_durable_barrier(cacheh); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init(cc, + cache_cfg, + io, + al, + "append-failure-poison", + hid, + platform_get_module_id()); + platform_assert_status_ok(rc); + + platform_assert_status_ok( + shard_log_iterator_create((cache *)cc, cfg, hid, segment, 0, &itor)); + platform_assert(!log_iterator_stream_complete(itor)); + platform_assert(log_iterator_can_next(itor)); + uint64 memtable_generation; + uint64 leaf_generation; + log_iterator_curr_generations(itor, &memtable_generation, &leaf_generation); + platform_assert(memtable_generation == 0); + platform_assert(leaf_generation == 0); + platform_assert_status_ok(log_iterator_next(itor)); + platform_assert(!log_iterator_can_next(itor)); + + log_iterator_deinit(itor); + shard_log_dec_ref((cache *)cc, &segment); + return 0; +} + +/* + * A begin ticket pins the handle, whose persistent mini-allocator reference + * also pins the stream's allocations. The owner may retire the handle and + * release its on-disk head before the waiter runs; the ticket must keep + * graduation safe and perform the final cleanup. + */ +static int +test_log_ticket_lifetime(cache *cc, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + log_handle *log; + platform_assert_status_ok(shard_log_create(cc, cfg, hid, &log)); + log_head head = log_get_head(log); + test_log_write_range(log, gen, hid, key_size, 0, 1); + allocator *al = cache_get_allocator(cc); + refcount refs_before_ticket = allocator_get_refcount(al, head.meta_addr); + + log_durable_ticket ticket; + platform_assert_status_ok(log_make_durable_begin(log, &ticket)); + platform_assert(ticket != 0); + platform_assert(allocator_get_refcount(al, head.meta_addr) + == refs_before_ticket, + "durability ticket changed the allocator refcount"); + + log_deinit(log); + platform_assert(allocator_get_refcount(al, head.meta_addr) + == refs_before_ticket, + "ticket did not keep the handle reference alive"); + shard_log_dec_ref(cc, &head); + platform_assert(allocator_get_refcount(al, head.meta_addr) + == refs_before_ticket - 1, + "head release did not leave only the handle reference"); + platform_assert_status_ok(log_make_durable_wait(log, ticket)); + platform_assert(allocator_get_refcount(al, head.meta_addr) == AL_FREE, + "final wait did not release the handle reference"); + return 0; +} + +typedef struct test_log_wait_actor { + log_handle *log; + log_durable_ticket ticket; + const bool32 *start; + platform_thread thread; + volatile bool32 ready; + platform_status rc; +} test_log_wait_actor; + +static void +test_log_wait_actor_run(void *arg) +{ + test_log_wait_actor *actor = arg; + __atomic_store_n(&actor->ready, TRUE, __ATOMIC_RELEASE); + while (!__atomic_load_n(actor->start, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + actor->rc = log_make_durable_wait(actor->log, actor->ticket); +} + +/* The last of several ticket waiters must uniquely destroy a retired handle. */ +static int +test_log_concurrent_ticket_lifetime(cache *cc, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + log_handle *log; + platform_assert_status_ok(shard_log_create(cc, cfg, hid, &log)); + log_head head = log_get_head(log); + test_log_write_range(log, gen, hid, key_size, 0, 1); + allocator *al = cache_get_allocator(cc); + refcount refs_before_waits = allocator_get_refcount(al, head.meta_addr); + + bool32 start = FALSE; + test_log_wait_actor actors[2] = {0}; + for (uint64 i = 0; i < ARRAY_SIZE(actors); i++) { + actors[i].log = log; + actors[i].start = &start; + actors[i].rc = STATUS_INVALID_STATE; + platform_assert_status_ok(log_make_durable_begin(log, &actors[i].ticket)); + platform_assert(actors[i].ticket != 0); + platform_assert_status_ok(platform_thread_create( + &actors[i].thread, FALSE, test_log_wait_actor_run, &actors[i], hid)); + } + for (uint64 i = 0; i < ARRAY_SIZE(actors); i++) { + platform_assert(test_log_wait_for_flag(&actors[i].ready)); + } + + log_deinit(log); + __atomic_store_n(&start, TRUE, __ATOMIC_RELEASE); + for (uint64 i = 0; i < ARRAY_SIZE(actors); i++) { + platform_assert_status_ok(platform_thread_join(&actors[i].thread)); + platform_assert_status_ok(actors[i].rc); + } + + platform_assert(allocator_get_refcount(al, head.meta_addr) + == refs_before_waits - 1, + "final concurrent wait did not release the handle ref"); + shard_log_dec_ref(cc, &head); + platform_assert(allocator_get_refcount(al, head.meta_addr) == AL_FREE, + "head release did not free the retired log"); + return 0; +} + /* * Sealing a stream and creating a fresh one must yield two distinct, * independently replayable segments. Reinitializing the cache after each forced @@ -248,13 +1372,14 @@ test_log_two_segments(clockcache *cc, const uint64 new_first = 2000, new_count = 16; log_head sealed, fresh; - log_handle *log = shard_log_create((cache *)cc, cfg, hid); - platform_assert(log != NULL); + log_handle *log; + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); sealed = log_get_head(log); // identity is fixed at creation test_log_write_range(log, gen, hid, key_size, old_first, old_count); - platform_status rc = log_seal(log); // frees log + platform_status rc = log_seal(log); platform_assert_status_ok(rc); + log_deinit(log); platform_assert(sealed.addr != 0); platform_assert(sealed.meta_addr != 0); @@ -270,17 +1395,17 @@ test_log_two_segments(clockcache *cc, test_log_verify_segment( (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); - // A fresh stream is a distinct segment: new mini allocator and new magic. - log = shard_log_create((cache *)cc, cfg, hid); - platform_assert(log != NULL); + // A fresh stream is a distinct segment: new mini allocator and new nonce. + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); fresh = log_get_head(log); test_log_write_range(log, gen, hid, key_size, new_first, new_count); - rc = log_seal(log); // frees log + rc = log_seal(log); platform_assert_status_ok(rc); + log_deinit(log); platform_assert(fresh.addr != 0); platform_assert(fresh.meta_addr != 0); platform_assert(sealed.meta_addr != fresh.meta_addr); - platform_assert(sealed.magic != fresh.magic); + platform_assert(!log_nonce_is_equal(sealed.nonce, fresh.nonce)); rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); @@ -296,8 +1421,8 @@ test_log_two_segments(clockcache *cc, test_log_verify_segment( (cache *)cc, cfg, &fresh, gen, hid, key_size, new_first, new_count); - log_dec_ref((cache *)cc, &sealed); - log_dec_ref((cache *)cc, &fresh); + shard_log_dec_ref((cache *)cc, &sealed); + shard_log_dec_ref((cache *)cc, &fresh); return 0; } @@ -312,10 +1437,11 @@ test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) message returned_message; char key_data[] = "large-log-key"; key skey = key_create(FALSE, sizeof(key_data) - 1, key_data); - uint64 value_len = 3 * cache_page_size(cc) + 123; + /* Exercise blob_writeback's whole-extent path and its partial tail. */ + uint64 value_len = cache_extent_size(cc) + 3 * cache_page_size(cc) + 123; - log_handle *logh = shard_log_create(cc, cfg, hid); - platform_assert(logh != NULL); + log_handle *logh; + platform_assert_status_ok(shard_log_create(cc, cfg, hid, &logh)); sealed = log_get_head(logh); // identity is fixed at creation merge_accumulator_init(&msg, hid); @@ -341,15 +1467,17 @@ test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) } merge_accumulator_deinit(&filler); - rc = log_seal(logh); // frees logh; identity captured above + rc = log_seal(logh); // identity captured above platform_assert_status_ok(rc); + log_deinit(logh); rc = cache_writeback_dirty(cc); platform_assert_status_ok(rc); rc = cache_durable_barrier(cc); platform_assert_status_ok(rc); - itor = shard_log_iterator_create(cc, cfg, hid, sealed); - platform_assert(itor != NULL); + platform_assert_status_ok( + shard_log_iterator_create(cc, cfg, hid, sealed, 0, &itor)); + platform_assert(log_iterator_stream_complete(itor)); platform_assert(log_iterator_can_next(itor)); log_iterator_curr(itor, &returned_key, &returned_message); @@ -360,7 +1488,240 @@ test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) log_iterator_deinit(itor); merge_accumulator_deinit(&msg); - log_dec_ref(cc, &sealed); + shard_log_dec_ref(cc, &sealed); + return 0; +} + +/* + * A checksum failure in one blob invalidates its whole log group, not merely + * that record. Earlier durable groups remain a replayable prefix, while an + * iterator whose generation bound excludes the blob need neither validate nor + * retain it. + */ +static int +test_log_blob_checksum_prefix(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid) +{ + cache *cacheh = (cache *)cc; + platform_status rc; + log_handle *log; + char key_data[] = "blob-checksum-prefix"; + key skey = key_create(FALSE, sizeof(key_data) - 1, key_data); + merge_accumulator msg; + merge_accumulator_init(&msg, hid); + + platform_assert_status_ok(shard_log_create(cacheh, cfg, hid, &log)); + log_head sealed = log_get_head(log); + + bool32 success = merge_accumulator_resize(&msg, 32); + platform_assert(success); + merge_accumulator_set_class(&msg, MESSAGE_TYPE_INSERT); + memset(merge_accumulator_data(&msg), 'A', merge_accumulator_length(&msg)); + platform_assert( + log_write(log, skey, merge_accumulator_to_message(&msg), 0, 0) == 0); + platform_assert_status_ok(log_make_durable(log)); + + success = merge_accumulator_resize(&msg, cache_page_size(cacheh) + 123); + platform_assert(success); + merge_accumulator_set_class(&msg, MESSAGE_TYPE_INSERT); + memset(merge_accumulator_data(&msg), 'B', merge_accumulator_length(&msg)); + platform_assert( + log_write(log, skey, merge_accumulator_to_message(&msg), 1, 0) == 0); + platform_assert_status_ok(log_seal(log)); + log_deinit(log); + + rc = cache_writeback_dirty(cacheh); + platform_assert_status_ok(rc); + rc = cache_durable_barrier(cacheh); + platform_assert_status_ok(rc); + + /* Locate a byte in the blob referenced by the second group. */ + log_iterator *itor; + platform_assert_status_ok( + shard_log_iterator_create(cacheh, cfg, hid, sealed, 0, &itor)); + platform_assert(log_iterator_stream_complete(itor)); + platform_assert(log_iterator_can_next(itor)); + platform_assert_status_ok(log_iterator_next(itor)); + platform_assert(log_iterator_can_next(itor)); + + key returned_key; + message returned_message; + log_iterator_curr(itor, &returned_key, &returned_message); + platform_assert(message_is_blob(returned_message)); + + slice sblob = message_slice(returned_message); + blob_page_iterator blob_itor; + rc = blob_page_iterator_init(cacheh, + &blob_itor, + sblob, + blob_length(sblob) - 1, + BLOB_PAGE_ITERATOR_MODE_NO_PREFETCH); + platform_assert_status_ok(rc); + uint64 ignored_offset; + slice ignored_data; + rc = blob_page_iterator_get_curr(&blob_itor, &ignored_offset, &ignored_data); + platform_assert_status_ok(rc); + uint64 corrupt_page_addr = blob_itor.fragment.addr; + uint64 corrupt_page_offset = blob_itor.fragment.offset; + blob_page_iterator_deinit(&blob_itor); + log_iterator_deinit(itor); + + page_handle *page = + cache_get(cacheh, corrupt_page_addr, TRUE, PAGE_TYPE_BLOB); + while (!cache_try_claim(cacheh, page)) { + cache_unget(cacheh, page); + page = cache_get(cacheh, corrupt_page_addr, TRUE, PAGE_TYPE_BLOB); + } + cache_lock(cacheh, page); + page->data[corrupt_page_offset] ^= 0x80; + cache_unlock(cacheh, page); + cache_unclaim(cacheh, page); + cache_unget(cacheh, page); + + rc = cache_writeback_dirty(cacheh); + platform_assert_status_ok(rc); + rc = cache_durable_barrier(cacheh); + platform_assert_status_ok(rc); + + /* Force replay to consult only the corrupted durable image. */ + clockcache_deinit(cc); + rc = clockcache_init(cc, + cache_cfg, + io, + al, + "blob-checksum-prefix", + hid, + platform_get_module_id()); + platform_assert_status_ok(rc); + + platform_assert_status_ok( + shard_log_iterator_create((cache *)cc, cfg, hid, sealed, 0, &itor)); + platform_assert(!log_iterator_stream_complete(itor)); + platform_assert(log_iterator_can_next(itor)); + uint64 memtable_generation; + uint64 leaf_generation; + log_iterator_curr_generations(itor, &memtable_generation, &leaf_generation); + platform_assert(memtable_generation == 0); + platform_assert(leaf_generation == 0); + platform_assert_status_ok(log_iterator_next(itor)); + platform_assert(!log_iterator_can_next(itor)); + + log_iterator_deinit(itor); + + /* + * Both records are already represented by a generation-2 checkpoint. The + * corrupted blob is therefore irrelevant, but group completeness and the + * stream terminator still have to be validated. Exact retained-byte sizing + * should make this an allocation-free empty iterator. + */ + platform_assert_status_ok( + shard_log_iterator_create((cache *)cc, cfg, hid, sealed, 2, &itor)); + platform_assert(log_iterator_stream_complete(itor)); + platform_assert(!log_iterator_can_next(itor)); + shard_log_iterator *shard_itor = (shard_log_iterator *)itor; + platform_assert(shard_itor->num_entries == 0); + platform_assert(shard_itor->entries == NULL); + platform_assert(shard_itor->contents == NULL); + log_iterator_deinit(itor); + + merge_accumulator_deinit(&msg); + shard_log_dec_ref((cache *)cc, &sealed); + return 0; +} + +/* + * Recovery must treat pages beyond a regular file's EOF as absent rather than + * relaxing all reads. Cover both shapes that motivated the range query: a + * fresh stream whose initial data extent has no page at all, and an extent + * whose last page write was torn at EOF. + */ +static int +test_log_recovery_at_eof(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + const char *filename) +{ + platform_status rc; + + log_handle *log; + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); + log_head empty = log_get_head(log); + + /* No log page has been written: EOF is exactly the initial extent base. */ + io_wait_all(io); + int sys_rc = truncate(filename, empty.addr); + platform_assert( + sys_rc == 0, "truncate(%s) failed with errno %d", filename, errno); + + log_iterator *itor; + platform_assert_status_ok( + shard_log_iterator_create((cache *)cc, cfg, hid, empty, 0, &itor)); + platform_assert(!log_iterator_can_next(itor)); + platform_assert(!log_iterator_stream_complete(itor)); + log_iterator_deinit(itor); + + log_deinit(log); + shard_log_dec_ref((cache *)cc, &empty); + + /* + * Four half-page values force at least four distinct log pages. Keep two + * complete pages and half of the next one; because the group's terminator + * was on a later page, replay must discard the group rather than return a + * prefix of it. + */ + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); + log_head partial = log_get_head(log); + + char key_data[] = "partial-log-extent"; + key skey = key_create(FALSE, sizeof(key_data) - 1, key_data); + merge_accumulator msg; + merge_accumulator_init(&msg, hid); + bool32 success = + merge_accumulator_resize(&msg, cache_page_size((cache *)cc) / 2); + platform_assert(success); + merge_accumulator_set_class(&msg, MESSAGE_TYPE_INSERT); + memset(merge_accumulator_data(&msg), 'P', merge_accumulator_length(&msg)); + + for (uint64 i = 0; i < 4; i++) { + int log_rc = + log_write(log, skey, merge_accumulator_to_message(&msg), i, 0); + platform_assert(log_rc == 0); + } + rc = log_seal(log); + platform_assert_status_ok(rc); + log_deinit(log); + merge_accumulator_deinit(&msg); + + rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + uint64 page_size = cache_page_size((cache *)cc); + clockcache_deinit(cc); + + uint64 partial_eof = partial.addr + 2 * page_size + page_size / 2; + sys_rc = truncate(filename, partial_eof); + platform_assert( + sys_rc == 0, "truncate(%s) failed with errno %d", filename, errno); + + rc = clockcache_init( + cc, cache_cfg, io, al, "partial-log-eof", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + + platform_assert_status_ok( + shard_log_iterator_create((cache *)cc, cfg, hid, partial, 0, &itor)); + platform_assert(!log_iterator_can_next(itor)); + platform_assert(!log_iterator_stream_complete(itor)); + log_iterator_deinit(itor); + shard_log_dec_ref((cache *)cc, &partial); + return 0; } @@ -400,6 +1761,152 @@ test_log_thread(void *arg) merge_accumulator_deinit(&msg); } +typedef struct test_log_pipeline_writer_params { + log_handle *log; + platform_thread thread; + test_message_generator *gen; + platform_heap_id hid; + uint64 key_size; + uint64 first; + uint64 count; + volatile bool32 *start; +} test_log_pipeline_writer_params; + +static void +test_log_pipeline_writer(void *arg) +{ + test_log_pipeline_writer_params *params = arg; + while (!__atomic_load_n(params->start, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(100); + } + test_log_write_range(params->log, + params->gen, + params->hid, + params->key_size, + params->first, + params->count); +} + +typedef struct test_log_pipeline_waiter_params { + log_handle *log; + platform_thread thread; + uint64 cuts; + volatile bool32 *start; + platform_status status; +} test_log_pipeline_waiter_params; + +static void +test_log_pipeline_waiter(void *arg) +{ + test_log_pipeline_waiter_params *params = arg; + while (!__atomic_load_n(params->start, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(100); + } + + params->status = STATUS_OK; + for (uint64 i = 0; i < params->cuts; i++) { + log_durable_ticket ticket; + params->status = log_make_durable_begin(params->log, &ticket); + if (!SUCCESS(params->status)) { + return; + } + /* Let other callers cut/stage later groups before this waiter drives. */ + platform_sleep_ns(1000); + params->status = log_make_durable_wait(params->log, ticket); + if (!SUCCESS(params->status)) { + return; + } + } +} + +/* Concurrent writers and split-phase waiters exercise reservation drains. */ +static int +test_log_concurrent_durability(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + enum { + NUM_WRITERS = 4, + NUM_WAITERS = 2, + }; + const uint64 entries_per_writer = 1024; + const uint64 cuts_per_waiter = 16; + volatile bool32 start = FALSE; + + log_handle *log; + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &log)); + log_head segment = log_get_head(log); + + test_log_pipeline_writer_params writers[NUM_WRITERS]; + test_log_pipeline_waiter_params waiters[NUM_WAITERS]; + for (uint64 i = 0; i < NUM_WRITERS; i++) { + writers[i] = (test_log_pipeline_writer_params){ + .log = log, + .gen = gen, + .hid = hid, + .key_size = key_size, + .first = i * entries_per_writer, + .count = entries_per_writer, + .start = &start, + }; + platform_assert_status_ok(platform_thread_create(&writers[i].thread, + FALSE, + test_log_pipeline_writer, + &writers[i], + hid)); + } + for (uint64 i = 0; i < NUM_WAITERS; i++) { + waiters[i] = (test_log_pipeline_waiter_params){ + .log = log, .cuts = cuts_per_waiter, .start = &start}; + platform_assert_status_ok(platform_thread_create(&waiters[i].thread, + FALSE, + test_log_pipeline_waiter, + &waiters[i], + hid)); + } + __atomic_store_n(&start, TRUE, __ATOMIC_RELEASE); + + for (uint64 i = 0; i < NUM_WRITERS; i++) { + platform_thread_join(&writers[i].thread); + } + for (uint64 i = 0; i < NUM_WAITERS; i++) { + platform_thread_join(&waiters[i].thread); + platform_assert_status_ok(waiters[i].status); + } + + platform_assert_status_ok(log_seal(log)); + log_deinit(log); + platform_status rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init(cc, + cache_cfg, + io, + al, + "concurrent-log-durability", + hid, + platform_get_module_id()); + platform_assert_status_ok(rc); + test_log_verify_segment((cache *)cc, + cfg, + &segment, + gen, + hid, + key_size, + 0, + NUM_WRITERS * entries_per_writer); + shard_log_dec_ref((cache *)cc, &segment); + return 0; +} + platform_status test_log_perf(cache *cc, shard_log_config *cfg, @@ -417,8 +1924,8 @@ test_log_perf(cache *cc, uint64 start_time; platform_status ret; - log_handle *logh = shard_log_create((cache *)cc, cfg, hid); - platform_assert(logh != NULL); + log_handle *logh; + platform_assert_status_ok(shard_log_create((cache *)cc, cfg, hid, &logh)); log_head sealed = log_get_head(logh); for (uint64 i = 0; i < num_threads; i++) { @@ -450,9 +1957,10 @@ test_log_perf(cache *cc, / platform_timestamp_elapsed(start_time)); cleanup: - // Seal (frees the handle) and release the segment's extents. - log_seal(logh); - log_dec_ref((cache *)cc, &sealed); + // Finish the stream, free the handle, and release the segment's extents. + platform_assert_status_ok(log_seal(logh)); + log_deinit(logh); + shard_log_dec_ref((cache *)cc, &sealed); platform_free(hid, params); return ret; @@ -571,9 +2079,126 @@ log_test(int argc, char *argv[]) platform_get_module_id()); platform_assert_status_ok(status); + rc = test_log_recovery_at_eof(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + system_cfg.io_cfg.filename); + platform_assert(rc == 0); + rc = test_log_large_message((cache *)cc, &system_cfg.log_cfg, hid); platform_assert(rc == 0); + rc = test_log_blob_checksum_prefix(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid); + platform_assert(rc == 0); + + rc = test_log_multiple_groups(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + + rc = test_log_pipelined_groups(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + + rc = test_log_concurrent_begin_handoff(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + + rc = test_log_begin_loses_install_claim(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + +#if SPLINTER_DEBUG + rc = test_log_nested_cutter_publication_handoff(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + + rc = test_log_begin_claim_precedes_seal(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); +#endif + + rc = test_log_seal_claim_precedes_begin(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + + rc = test_log_append_failure_poison(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + + rc = test_log_ticket_lifetime( + (cache *)cc, &system_cfg.log_cfg, hid, &gen, workload_cfg.key_size); + platform_assert(rc == 0); + + rc = test_log_concurrent_ticket_lifetime( + (cache *)cc, &system_cfg.log_cfg, hid, &gen, workload_cfg.key_size); + platform_assert(rc == 0); + + rc = test_log_concurrent_durability(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + rc = test_log_two_segments(cc, &system_cfg.cache_cfg, io, diff --git a/tests/functional/scan_benchmark.c b/tests/functional/scan_benchmark.c index dc8f2a9e..e25c2e3e 100644 --- a/tests/functional/scan_benchmark.c +++ b/tests/functional/scan_benchmark.c @@ -438,7 +438,7 @@ scan_benchmark_load_database(const splinterdb_config *cfg, uint8 *value_buf = TYPED_ARRAY_ZALLOC(platform_get_heap_id(), value_buf, value_size); if (value_buf == NULL) { - splinterdb_close(&kvs); + splinterdb_close(&kvs, FALSE); return scan_benchmark_status_to_int(STATUS_NO_MEMORY); } @@ -489,7 +489,7 @@ scan_benchmark_load_database(const splinterdb_config *cfg, } platform_free(platform_get_heap_id(), value_buf); - splinterdb_close(&kvs); + splinterdb_close(&kvs, FALSE); return rc; } @@ -526,7 +526,7 @@ scan_benchmark_run_optimize(const splinterdb_config *cfg) io_print_stats((io_handle *)splinterdb_get_io_handle(kvs), Platform_default_log_handle); - splinterdb_close(&kvs); + splinterdb_close(&kvs, FALSE); return rc; } @@ -550,7 +550,7 @@ scan_benchmark_run_scan(const splinterdb_config *cfg, rc = splinterdb_iterator_init( kvs, &iter, scan_benchmark_start_comparison(backwards_scan), NULL_SLICE); if (rc != 0) { - splinterdb_close(&kvs); + splinterdb_close(&kvs, FALSE); return rc; } @@ -610,7 +610,7 @@ scan_benchmark_run_scan(const splinterdb_config *cfg, Platform_default_log_handle); splinterdb_iterator_deinit(iter); - splinterdb_close(&kvs); + splinterdb_close(&kvs, FALSE); return rc; } @@ -639,7 +639,7 @@ scan_benchmark_run_repeated_scans(const splinterdb_config *cfg, if (effective_scan_length == 0) { platform_error_log("scan_benchmark: repeated scans require a non-zero " "scan length or --num-inserts\n"); - splinterdb_close(&kvs); + splinterdb_close(&kvs, FALSE); return EINVAL; } @@ -692,7 +692,7 @@ scan_benchmark_run_repeated_scans(const splinterdb_config *cfg, scan_benchmark_start_comparison(backwards_scan), start_key); if (rc != 0) { - splinterdb_close(&kvs); + splinterdb_close(&kvs, FALSE); return rc; } @@ -731,7 +731,7 @@ scan_benchmark_run_repeated_scans(const splinterdb_config *cfg, rc = splinterdb_iterator_status(iter); if (rc != 0) { splinterdb_iterator_deinit(iter); - splinterdb_close(&kvs); + splinterdb_close(&kvs, FALSE); return rc; } @@ -779,7 +779,7 @@ scan_benchmark_run_repeated_scans(const splinterdb_config *cfg, io_print_stats((io_handle *)splinterdb_get_io_handle(kvs), Platform_default_log_handle); - splinterdb_close(&kvs); + splinterdb_close(&kvs, FALSE); return rc; } diff --git a/tests/functional/test.h b/tests/functional/test.h index 11bbd1bf..d3822c1a 100644 --- a/tests/functional/test.h +++ b/tests/functional/test.h @@ -303,6 +303,12 @@ test_config_init(system_config *system_cfg, // OUT uint64 checkpoint_log_size = master_cfg->checkpoint_log_size != 0 ? master_cfg->checkpoint_log_size : master_cfg->cache_capacity; + uint64 checkpoint_log_grace_bytes = + master_cfg->checkpoint_log_grace_bytes != 0 + ? master_cfg->checkpoint_log_grace_bytes + : (master_cfg->memtable_capacity > UINT64_MAX / 2 + ? UINT64_MAX + : 2 * master_cfg->memtable_capacity); rc = core_config_init(&system_cfg->splinter_cfg, &system_cfg->cache_cfg.super, @@ -314,6 +320,7 @@ test_config_init(system_config *system_cfg, // OUT master_cfg->prefetch_budget, master_cfg->use_log, checkpoint_log_size, + checkpoint_log_grace_bytes, master_cfg->use_stats, master_cfg->verbose_logging_enabled, master_cfg->log_handle); diff --git a/tests/functional/ycsb_test.c b/tests/functional/ycsb_test.c index 7256f8ac..e598e7fb 100644 --- a/tests/functional/ycsb_test.c +++ b/tests/functional/ycsb_test.c @@ -1324,7 +1324,7 @@ ycsb_test(int argc, char *argv[]) run_all_ycsb_phases(&spl, phases, nphases, &ts, hid); - core_unmount(&spl); + platform_assert_status_ok(core_unmount(&spl, FALSE)); clockcache_deinit(cc); platform_free(hid, cc); // core_unmount() already persisted the map and published the superblock. diff --git a/tests/unit/large_inserts_stress_test.c b/tests/unit/large_inserts_stress_test.c index b0a07cf0..2b2a293f 100644 --- a/tests/unit/large_inserts_stress_test.c +++ b/tests/unit/large_inserts_stress_test.c @@ -161,7 +161,7 @@ CTEST_TEARDOWN(large_inserts_stress) { // Only parent process should tear down Splinter. if (data->am_parent) { - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); platform_heap_destroy(&data->hid); } platform_deregister_thread(); diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index a3301f89..28f221c4 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -22,11 +22,15 @@ * ----------------------------------------------------------------------------- */ #include "core.h" +#include "shard_log.h" +#include "blob_build.h" #include "clockcache.h" #include "allocator.h" +#include "mini_allocator.h" #include "rc_allocator.h" #include "task.h" #include "platform_threads.h" +#include "platform_sleep.h" #include "functional/test.h" #include "functional/test_async.h" #include "test_common.h" @@ -47,6 +51,514 @@ typedef struct trunk_shadow { writable_buffer data; } trunk_shadow; +/* + * Test-only durable-barrier fault injection for checkpoint retries. + * + * The whole fixture shares one io_handle, so temporarily replacing its ops + * table reaches log, cache, and superblock barriers without changing any of + * their production interfaces. Tests install only one injector at a time and + * keep it installed until the task system is quiescent. + */ +typedef struct checkpoint_barrier_fault { + io_handle *io; + const io_ops *saved_ops; + io_ops fault_ops; + uint64 skip_barriers; + uint64 fail_barriers; + uint64 barriers; + uint64 block_barrier; + bool32 enabled; + bool32 block_enabled; + bool32 block_entered; + bool32 block_released; + bool32 installed; +} checkpoint_barrier_fault; + +/* + * Long enough to cover both transition-triggered and explicit advance attempts, + * finite so an implementation which mistakenly swallows the errors eventually + * escapes and fails the test instead of hanging the test suite forever. + */ +#define CHECKPOINT_BARRIER_FAULT_COUNT 32 + +static checkpoint_barrier_fault *active_checkpoint_barrier_fault; + +static platform_status +checkpoint_fault_durable_barrier(io_handle *io) +{ + checkpoint_barrier_fault *fault = + __atomic_load_n(&active_checkpoint_barrier_fault, __ATOMIC_ACQUIRE); + platform_assert(fault != NULL && fault->installed && fault->io == io); + + if (__atomic_load_n(&fault->enabled, __ATOMIC_ACQUIRE)) { + uint64 barrier = + __atomic_fetch_add(&fault->barriers, 1, __ATOMIC_RELAXED); + if (__atomic_load_n(&fault->block_enabled, __ATOMIC_ACQUIRE) + && barrier == fault->block_barrier) + { + __atomic_store_n(&fault->block_entered, TRUE, __ATOMIC_RELEASE); + while (!__atomic_load_n(&fault->block_released, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + } + if (barrier >= fault->skip_barriers + && barrier - fault->skip_barriers < fault->fail_barriers) + { + return STATUS_IO_ERROR; + } + } + + return fault->saved_ops->durable_barrier(io); +} + +static void +checkpoint_barrier_fault_install(checkpoint_barrier_fault *fault, io_handle *io) +{ + platform_assert(active_checkpoint_barrier_fault == NULL); + platform_assert(!fault->installed); + + fault->io = io; + fault->saved_ops = io->ops; + fault->fault_ops = *io->ops; + fault->fault_ops.durable_barrier = checkpoint_fault_durable_barrier; + fault->installed = TRUE; + + __atomic_store_n(&active_checkpoint_barrier_fault, fault, __ATOMIC_RELEASE); + io->ops = &fault->fault_ops; +} + +static void +checkpoint_barrier_fault_arm(checkpoint_barrier_fault *fault, + uint64 skip_barriers) +{ + platform_assert(fault->installed); + __atomic_store_n(&fault->enabled, FALSE, __ATOMIC_RELEASE); + __atomic_store_n(&fault->block_released, TRUE, __ATOMIC_RELEASE); + __atomic_store_n(&fault->block_enabled, FALSE, __ATOMIC_RELEASE); + fault->skip_barriers = skip_barriers; + fault->fail_barriers = CHECKPOINT_BARRIER_FAULT_COUNT; + __atomic_store_n(&fault->barriers, 0, __ATOMIC_RELAXED); + __atomic_store_n(&fault->enabled, TRUE, __ATOMIC_RELEASE); +} + +/* Block one selected successful device barrier until the test releases it. */ +static void +checkpoint_barrier_fault_block(checkpoint_barrier_fault *fault, + uint64 block_barrier) +{ + platform_assert(fault->installed); + __atomic_store_n(&fault->enabled, FALSE, __ATOMIC_RELEASE); + __atomic_store_n(&fault->block_released, TRUE, __ATOMIC_RELEASE); + __atomic_store_n(&fault->block_enabled, FALSE, __ATOMIC_RELEASE); + + fault->skip_barriers = 0; + fault->fail_barriers = 0; + fault->block_barrier = block_barrier; + __atomic_store_n(&fault->barriers, 0, __ATOMIC_RELAXED); + __atomic_store_n(&fault->block_entered, FALSE, __ATOMIC_RELAXED); + __atomic_store_n(&fault->block_released, FALSE, __ATOMIC_RELAXED); + __atomic_store_n(&fault->block_enabled, TRUE, __ATOMIC_RELEASE); + __atomic_store_n(&fault->enabled, TRUE, __ATOMIC_RELEASE); +} + +static void +checkpoint_barrier_fault_release(checkpoint_barrier_fault *fault) +{ + __atomic_store_n(&fault->block_released, TRUE, __ATOMIC_RELEASE); +} + +static void +checkpoint_barrier_fault_disable(checkpoint_barrier_fault *fault) +{ + checkpoint_barrier_fault_release(fault); + __atomic_store_n(&fault->block_enabled, FALSE, __ATOMIC_RELEASE); + __atomic_store_n(&fault->enabled, FALSE, __ATOMIC_RELEASE); +} + +static void +checkpoint_barrier_fault_uninstall(checkpoint_barrier_fault *fault) +{ + if (!fault->installed) { + return; + } + + checkpoint_barrier_fault_disable(fault); + io_wait_all(fault->io); + fault->io->ops = fault->saved_ops; + __atomic_store_n(&active_checkpoint_barrier_fault, NULL, __ATOMIC_RELEASE); + fault->installed = FALSE; + fault->io = NULL; + fault->saved_ops = NULL; +} + +/* + * Pause the first core insert after its leaf-lock callback has reserved a log + * group and made the memtable mutation visible, but before the reserved append + * consumes that reservation. Later writes pass through so a test can prove + * that the blocked reservation does not exclude unrelated writers. + */ +typedef struct core_log_write_block { + log_handle *log; + const log_ops *saved_ops; + log_ops blocked_ops; + uint64 calls; + bool32 entered; + bool32 released; + bool32 installed; +} core_log_write_block; + +static core_log_write_block *active_core_log_write_block; + +static int +core_log_write_reserved_blocked(log_write_token *token, + key tuple_key, + message data, + uint64 memtable_generation, + uint64 leaf_generation) +{ + core_log_write_block *block = + __atomic_load_n(&active_core_log_write_block, __ATOMIC_ACQUIRE); + platform_assert(block != NULL && block->installed + && block->log == token->log); + + uint64 call = __atomic_fetch_add(&block->calls, 1, __ATOMIC_RELAXED); + if (call == 0) { + __atomic_store_n(&block->entered, TRUE, __ATOMIC_RELEASE); + while (!__atomic_load_n(&block->released, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + } + return block->saved_ops->write_reserved( + token, tuple_key, data, memtable_generation, leaf_generation); +} + +static void +core_log_write_block_install(core_log_write_block *block, log_handle *log) +{ + platform_assert(active_core_log_write_block == NULL); + platform_assert(!block->installed); + + block->log = log; + block->saved_ops = log->ops; + block->blocked_ops = *log->ops; + block->blocked_ops.write_reserved = core_log_write_reserved_blocked; + block->calls = 0; + block->entered = FALSE; + block->released = FALSE; + block->installed = TRUE; + __atomic_store_n(&active_core_log_write_block, block, __ATOMIC_RELEASE); + log->ops = &block->blocked_ops; +} + +static void +core_log_write_block_release(core_log_write_block *block) +{ + __atomic_store_n(&block->released, TRUE, __ATOMIC_RELEASE); +} + +static void +core_log_write_block_uninstall(core_log_write_block *block) +{ + if (!block->installed) { + return; + } + core_log_write_block_release(block); + block->log->ops = block->saved_ops; + __atomic_store_n(&active_core_log_write_block, NULL, __ATOMIC_RELEASE); + block->installed = FALSE; + block->log = NULL; + block->saved_ops = NULL; +} + +/* + * Replace one valid core update's log message with a malformed blob only after + * the memtable mutation has linearized. The concrete log consumes the real + * reservation and records the append failure in its group, while the memtable + * retains the caller's original value. + */ +typedef struct core_log_append_fault { + log_handle *log; + const log_ops *saved_ops; + log_ops fault_ops; + uint64 calls; + bool32 installed; +} core_log_append_fault; + +static core_log_append_fault *active_core_log_append_fault; + +static int +core_log_write_reserved_invalid_blob(log_write_token *token, + key tuple_key, + message data, + uint64 memtable_generation, + uint64 leaf_generation) +{ + core_log_append_fault *fault = + __atomic_load_n(&active_core_log_append_fault, __ATOMIC_ACQUIRE); + platform_assert(fault != NULL && fault->installed + && fault->log == token->log); + (void)data; + + __atomic_fetch_add(&fault->calls, 1, __ATOMIC_RELAXED); + blob invalid_blob = { + .length = 0, + .checksum = {0}, + .format = BLOB_FORMAT + 1, + }; + message invalid_msg = + message_create(MESSAGE_TYPE_INSERT, + ((shard_log *)fault->log)->cc, + slice_create(sizeof(invalid_blob), &invalid_blob)); + return fault->saved_ops->write_reserved( + token, tuple_key, invalid_msg, memtable_generation, leaf_generation); +} + +static void +core_log_append_fault_install(core_log_append_fault *fault, log_handle *log) +{ + platform_assert(active_core_log_append_fault == NULL); + platform_assert(!fault->installed); + + fault->log = log; + fault->saved_ops = log->ops; + fault->fault_ops = *log->ops; + fault->fault_ops.write_reserved = core_log_write_reserved_invalid_blob; + fault->calls = 0; + fault->installed = TRUE; + __atomic_store_n(&active_core_log_append_fault, fault, __ATOMIC_RELEASE); + log->ops = &fault->fault_ops; +} + +static void +core_log_append_fault_uninstall(core_log_append_fault *fault) +{ + if (!fault->installed) { + return; + } + + fault->log->ops = fault->saved_ops; + __atomic_store_n(&active_core_log_append_fault, NULL, __ATOMIC_RELEASE); + fault->installed = FALSE; + fault->log = NULL; + fault->saved_ops = NULL; +} + +#define CORE_DURABLE_BARRIER_TEST_TIMEOUT_NS SEC_TO_NSEC(10) + +static bool32 +core_durable_barrier_test_wait(const bool32 *flag) +{ + timestamp start = platform_get_timestamp(); + while (!__atomic_load_n(flag, __ATOMIC_ACQUIRE) + && platform_timestamp_elapsed(start) + < CORE_DURABLE_BARRIER_TEST_TIMEOUT_NS) + { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return __atomic_load_n(flag, __ATOMIC_ACQUIRE); +} + +static bool32 +core_durable_barrier_test_wait_for_handle_refs(shard_log *log, uint64 target) +{ + timestamp start = platform_get_timestamp(); + while (platform_timestamp_elapsed(start) + < CORE_DURABLE_BARRIER_TEST_TIMEOUT_NS) + { + uint64 refs = __atomic_load_n(&log->handle_refs, __ATOMIC_RELAXED); + if (refs >= target) { + return TRUE; + } + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return FALSE; +} + +static bool32 +core_durable_barrier_test_wait_for_io_barriers(checkpoint_barrier_fault *fault, + uint64 target) +{ + timestamp start = platform_get_timestamp(); + while (__atomic_load_n(&fault->barriers, __ATOMIC_ACQUIRE) < target + && platform_timestamp_elapsed(start) + < CORE_DURABLE_BARRIER_TEST_TIMEOUT_NS) + { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return __atomic_load_n(&fault->barriers, __ATOMIC_ACQUIRE) >= target; +} + +/* Take a stable frontier snapshot without dereferencing the unhazarded group. + */ +static bool32 +core_durable_barrier_test_try_cut_frontier(shard_log *log, + log_durable_ticket *frontier_out) +{ + uint64 install_before = + __atomic_load_n(&log->install.state, __ATOMIC_SEQ_CST); + shard_log_group *accepting = + __atomic_load_n(&log->accepting.group, __ATOMIC_SEQ_CST); + uint64 accepting_id = __atomic_load_n(&log->accepting.id, __ATOMIC_SEQ_CST); + uint64 install_after = + __atomic_load_n(&log->install.state, __ATOMIC_SEQ_CST); + + if (install_before != install_after) { + return FALSE; + } + + if (install_after & SHARD_LOG_INSTALL_TERMINAL_BIT) { + if (accepting != NULL) { + return FALSE; + } + *frontier_out = install_after & SHARD_LOG_INSTALL_ID_MASK; + return TRUE; + } + + if (accepting == NULL || install_after != accepting_id) { + return FALSE; + } + platform_assert(accepting_id >= SHARD_LOG_FIRST_GROUP_ID); + *frontier_out = accepting_id - 1; + return TRUE; +} + +static bool32 +core_durable_barrier_test_wait_for_live_log_cut(shard_log *log) +{ + timestamp start = platform_get_timestamp(); + while (platform_timestamp_elapsed(start) + < CORE_DURABLE_BARRIER_TEST_TIMEOUT_NS) + { + log_durable_ticket cut_frontier; + if (core_durable_barrier_test_try_cut_frontier(log, &cut_frontier) + && cut_frontier != 0) + { + return TRUE; + } + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return FALSE; +} + +static bool32 +core_durable_barrier_test_wait_for_live_log_durable(shard_log *log) +{ + timestamp start = platform_get_timestamp(); + while (platform_timestamp_elapsed(start) + < CORE_DURABLE_BARRIER_TEST_TIMEOUT_NS) + { + log_durable_ticket cut_frontier; + if (core_durable_barrier_test_try_cut_frontier(log, &cut_frontier) + && cut_frontier != 0 + && __atomic_load_n(&log->durable_ticket, __ATOMIC_ACQUIRE) + >= cut_frontier) + { + return TRUE; + } + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return FALSE; +} + +typedef struct core_durable_barrier_thread_args { + core_handle *spl; + platform_status rc; + bool32 started; + bool32 done; +} core_durable_barrier_thread_args; + +static void +core_durable_barrier_test_thread(void *arg) +{ + core_durable_barrier_thread_args *args = arg; + __atomic_store_n(&args->started, TRUE, __ATOMIC_RELEASE); + args->rc = core_durable_barrier(args->spl); + __atomic_store_n(&args->done, TRUE, __ATOMIC_RELEASE); +} + +typedef struct core_checkpoint_thread_args { + core_handle *spl; + platform_status rc; + bool32 done; +} core_checkpoint_thread_args; + +static void +core_checkpoint_test_thread(void *arg) +{ + core_checkpoint_thread_args *args = arg; + args->rc = core_checkpoint(args->spl, 0); + __atomic_store_n(&args->done, TRUE, __ATOMIC_RELEASE); +} + +typedef struct core_durable_barrier_insert_args { + core_handle *spl; + key tuple_key; + message msg; + bool32 *start; + bool32 *release; + threadid tid; + platform_status rc; + bool32 ready; + bool32 done; +} core_durable_barrier_insert_args; + +static void +core_durable_barrier_insert_thread(void *arg) +{ + core_durable_barrier_insert_args *args = arg; + args->tid = platform_get_tid(); + __atomic_store_n(&args->ready, TRUE, __ATOMIC_RELEASE); + while (args->start != NULL + && !__atomic_load_n(args->start, __ATOMIC_ACQUIRE)) + { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + args->rc = core_insert(args->spl, args->tuple_key, args->msg, NULL); + __atomic_store_n(&args->done, TRUE, __ATOMIC_RELEASE); + while (args->release != NULL + && !__atomic_load_n(args->release, __ATOMIC_ACQUIRE)) + { + platform_sleep_ns(USEC_TO_NSEC(50)); + } +} + +static bool32 +core_durable_barrier_test_wait_for_insert_threads( + core_durable_barrier_insert_args *args, + uint64 num_args, + bool32 wait_for_done) +{ + timestamp start = platform_get_timestamp(); + while (platform_timestamp_elapsed(start) + < CORE_DURABLE_BARRIER_TEST_TIMEOUT_NS) + { + bool32 all_reached = TRUE; + for (uint64 i = 0; i < num_args; i++) { + const bool32 *flag = wait_for_done ? &args[i].done : &args[i].ready; + all_reached &= __atomic_load_n(flag, __ATOMIC_ACQUIRE); + } + if (all_reached) { + return TRUE; + } + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return FALSE; +} + +static bool32 +checkpoint_record_names_log(superblock_log_head recorded, log_head log) +{ + return !SUPERBLOCK_NO_LOG(recorded) && log_head_is_equal(recorded.head, log); +} + +static bool32 +checkpoint_records_name_same_log(superblock_log_head left, + superblock_log_head right) +{ + return !SUPERBLOCK_NO_LOG(left) && !SUPERBLOCK_NO_LOG(right) + && log_head_is_equal(left.head, right.head); +} + /* Function prototypes */ static uint64 splinter_do_inserts(void *datap, @@ -88,12 +600,13 @@ CTEST_DATA(splinter) rc_allocator al; // Following get setup pointing to allocated memory - system_config *system_cfg; - test_workload_config *workload_cfg; - io_handle *io; - clockcache *clock_cache; - task_system tasks; - test_message_generator gen; + system_config *system_cfg; + test_workload_config *workload_cfg; + io_handle *io; + clockcache *clock_cache; + task_system tasks; + test_message_generator gen; + checkpoint_barrier_fault checkpoint_fault; // Test execution related configuration test_exec_config test_exec_cfg; @@ -133,6 +646,7 @@ CTEST_SETUP(splinter) TYPED_ARRAY_MALLOC(data->hid, data->workload_cfg, num_tables); ZERO_STRUCT(data->test_exec_cfg); + ZERO_STRUCT(data->checkpoint_fault); rc = test_parse_args_n(data->system_cfg, &data->test_exec_cfg, @@ -191,6 +705,8 @@ CTEST_SETUP(splinter) */ CTEST_TEARDOWN(splinter) { + checkpoint_barrier_fault_uninstall(&data->checkpoint_fault); + clockcache_deinit(data->clock_cache); platform_free(data->hid, data->clock_cache); @@ -213,6 +729,365 @@ CTEST_TEARDOWN(splinter) platform_deregister_thread(); } +static void +blob_checksum_test_fill(writable_buffer *data, uint64 length) +{ + uint8 *bytes = writable_buffer_data(data); + for (uint64 i = 0; i < length; i++) { + bytes[i] = (uint8)(131 * i + i / 7 + length); + } +} + +static platform_status +blob_checksum_test_roundtrip(cache *cc, + slice descriptor, + slice expected, + writable_buffer *materialized) +{ + platform_status rc = blob_materialize_full(cc, descriptor, materialized); + if (!SUCCESS(rc)) { + return rc; + } + if (writable_buffer_length(materialized) != slice_length(expected)) { + return STATUS_TEST_FAILED; + } + if (slice_length(expected) != 0 + && memcmp(writable_buffer_data(materialized), + slice_data(expected), + slice_length(expected)) + != 0) + { + return STATUS_TEST_FAILED; + } + return STATUS_OK; +} + +static platform_status +blob_checksum_test_writeback(cache *cc, slice descriptor) +{ + writeback_set set; + writeback_set_init(&set, cc, platform_get_heap_id()); + + platform_status rc = blob_writeback(cc, descriptor, &set); + platform_status wait_rc = writeback_set_wait(&set); + if (SUCCESS(rc)) { + rc = wait_rc; + } + writeback_set_deinit(&set); + return rc; +} + +static platform_status +blob_checksum_test_build_and_check(const blob_build_config *cfg, + cache *cc, + mini_allocator *mini, + uint64 length, + writable_buffer *data, + writable_buffer *descriptor, + writable_buffer *materialized) +{ + platform_status rc = writable_buffer_resize(data, length); + if (!SUCCESS(rc)) { + return rc; + } + blob_checksum_test_fill(data, length); + + rc = blob_build(cfg, cc, mini, writable_buffer_to_slice(data), descriptor); + if (!SUCCESS(rc)) { + return rc; + } + + slice sblob = writable_buffer_to_slice(descriptor); + rc = blob_checksum_test_writeback(cc, sblob); + if (!SUCCESS(rc)) { + return rc; + } + + const blob *blobby = slice_data(sblob); + if (slice_length(sblob) < sizeof(*blobby) || blobby->format != BLOB_FORMAT) { + return STATUS_TEST_FAILED; + } + + parsed_blob pblob; + parse_blob( + cache_extent_size(cc), cache_page_size(cc), slice_data(sblob), &pblob); + uint64 num_addrs = pblob.num_extents; + for (uint64 i = 0; i < ARRAY_SIZE(pblob.leftovers); i++) { + if (pblob.leftovers[i].length == 0) { + break; + } + num_addrs++; + } + if (slice_length(sblob) != sizeof(blob) + num_addrs * sizeof(uint64)) { + return STATUS_TEST_FAILED; + } + + rc = blob_validate(cc, sblob); + if (!SUCCESS(rc)) { + return rc; + } + return blob_checksum_test_roundtrip( + cc, sblob, writable_buffer_to_slice(data), materialized); +} + +static platform_status +blob_checksum_test_mini_init(cache *cc, mini_allocator *mini, uint64 *meta_head) +{ + allocator *al = cache_get_allocator(cc); + page_type types[NUM_BLOB_BATCHES] = { + PAGE_TYPE_BLOB, + PAGE_TYPE_BLOB, + PAGE_TYPE_BLOB, + }; + platform_status rc = allocator_alloc(al, meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + return rc; + } + mini_init_with_types( + mini, cc, *meta_head, 0, NUM_BLOB_BATCHES, PAGE_TYPE_MISC, types); + return STATUS_OK; +} + +static platform_status +blob_checksum_test_mini_deinit(cache *cc, + mini_allocator *mini, + uint64 meta_head) +{ + mini_release(mini); + return mini_dec_ref(cc, meta_head, PAGE_TYPE_MISC) == 0 ? STATUS_OK + : STATUS_TEST_FAILED; +} + +CTEST2(splinter, test_blob_checksums) +{ + cache *cc = (cache *)data->clock_cache; + blob_build_config cfg = { + .extent_batch = 0, + .page_batch = 1, + .subpage_batch = 2, + .alignment = 0, + }; + + writable_buffer source_data; + writable_buffer descriptor; + writable_buffer materialized; + writable_buffer checked_clone; + writable_buffer invalid_clone; + writable_buffer_init(&source_data, data->hid); + writable_buffer_init(&descriptor, data->hid); + writable_buffer_init(&materialized, data->hid); + writable_buffer_init(&checked_clone, data->hid); + writable_buffer_init(&invalid_clone, data->hid); + + mini_allocator source_mini; + mini_allocator checked_clone_mini; + bool32 source_mini_live = FALSE; + bool32 checked_clone_mini_live = FALSE; + uint64 source_meta_head = 0; + uint64 checked_clone_meta_head = 0; + + platform_status rc = + blob_checksum_test_mini_init(cc, &source_mini, &source_meta_head); + if (!SUCCESS(rc)) { + goto cleanup; + } + source_mini_live = TRUE; + + uint64 page_size = cache_page_size(cc); + uint64 extent_size = cache_extent_size(cc); + uint64 lengths[] = { + 0, + 1, + page_size / 2 + 1, + page_size - 1, + page_size + page_size / 2, + extent_size - 1, + extent_size, + extent_size + page_size / 2, + 2 * extent_size + page_size + 17, + }; + for (uint64 i = 0; i < ARRAY_SIZE(lengths); i++) { + rc = blob_checksum_test_build_and_check(&cfg, + cc, + &source_mini, + lengths[i], + &source_data, + &descriptor, + &materialized); + if (!SUCCESS(rc)) { + goto cleanup; + } + } + + /* Leave one full extent plus a separately allocated tail for cloning. */ + uint64 clone_length = extent_size + page_size / 2 + 17; + rc = blob_checksum_test_build_and_check(&cfg, + cc, + &source_mini, + clone_length, + &source_data, + &descriptor, + &materialized); + if (!SUCCESS(rc)) { + goto cleanup; + } + + rc = blob_checksum_test_mini_init( + cc, &checked_clone_mini, &checked_clone_meta_head); + if (!SUCCESS(rc)) { + goto cleanup; + } + checked_clone_mini_live = TRUE; + rc = blob_clone(&cfg, + cc, + &checked_clone_mini, + writable_buffer_to_slice(&descriptor), + &checked_clone); + if (!SUCCESS(rc)) { + goto cleanup; + } + + const blob *source_blob = writable_buffer_data(&descriptor); + const blob *clone_blob = writable_buffer_data(&checked_clone); + if (source_blob->format != BLOB_FORMAT || clone_blob->format != BLOB_FORMAT) + { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + checksum128 source_checksum = source_blob->checksum; + checksum128 clone_checksum = clone_blob->checksum; + if (!platform_checksum_is_equal(source_checksum, clone_checksum)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = blob_checksum_test_writeback(cc, + writable_buffer_to_slice(&checked_clone)); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = blob_validate(cc, writable_buffer_to_slice(&checked_clone)); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = blob_checksum_test_roundtrip(cc, + writable_buffer_to_slice(&checked_clone), + writable_buffer_to_slice(&source_data), + &materialized); + if (!SUCCESS(rc)) { + goto cleanup; + } + + message inline_msg = message_create( + MESSAGE_TYPE_INSERT, NULL, writable_buffer_to_slice(&source_data)); + message blob_msg = message_create( + MESSAGE_TYPE_INSERT, cc, writable_buffer_to_slice(&descriptor)); + if (!SUCCESS(message_validate(inline_msg)) + || !SUCCESS(message_validate(blob_msg))) + { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + /* Unknown descriptor formats must be rejected, not treated as legacy. */ + blob *mutable_blob = writable_buffer_data(&descriptor); + uint16 expected_format = mutable_blob->format; + mutable_blob->format = BLOB_FORMAT + 1; + slice invalid = writable_buffer_to_slice(&descriptor); + platform_status validate_rc = blob_validate(cc, invalid); + platform_status materialize_rc = + blob_materialize_full(cc, invalid, &materialized); + platform_status clone_rc = + blob_clone(&cfg, cc, &checked_clone_mini, invalid, &invalid_clone); + mutable_blob->format = expected_format; + if (!STATUS_IS_EQ(validate_rc, STATUS_INVALID_STATE) + || !STATUS_IS_EQ(materialize_rc, STATUS_INVALID_STATE) + || !STATUS_IS_EQ(clone_rc, STATUS_INVALID_STATE)) + { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + /* Corrupt the clone's private tail, leaving its shared full extent alone. */ + blob_page_iterator iter; + rc = blob_page_iterator_init(cc, + &iter, + writable_buffer_to_slice(&checked_clone), + clone_length - 1, + BLOB_PAGE_ITERATOR_MODE_NO_PREFETCH); + if (!SUCCESS(rc)) { + goto cleanup; + } + uint64 corrupt_offset; + slice corrupt_data; + rc = blob_page_iterator_get_curr(&iter, &corrupt_offset, &corrupt_data); + if (!SUCCESS(rc)) { + blob_page_iterator_deinit(&iter); + goto cleanup; + } + if (corrupt_offset != clone_length - 1 || slice_length(corrupt_data) == 0) { + blob_page_iterator_deinit(&iter); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + uint64 corrupt_page_addr = iter.fragment.addr; + uint64 corrupt_page_offset = iter.fragment.offset; + blob_page_iterator_deinit(&iter); + + page_handle *page = cache_get(cc, corrupt_page_addr, TRUE, PAGE_TYPE_BLOB); + if (page == NULL) { + rc = STATUS_IO_ERROR; + goto cleanup; + } + while (!cache_try_claim(cc, page)) { + cache_unget(cc, page); + page = cache_get(cc, corrupt_page_addr, TRUE, PAGE_TYPE_BLOB); + } + cache_lock(cc, page); + page->data[corrupt_page_offset] ^= 0x80; + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + + rc = blob_validate(cc, writable_buffer_to_slice(&checked_clone)); + if (!STATUS_IS_EQ(rc, STATUS_IO_ERROR)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + blob_msg = message_create( + MESSAGE_TYPE_INSERT, cc, writable_buffer_to_slice(&checked_clone)); + rc = message_validate(blob_msg); + if (!STATUS_IS_EQ(rc, STATUS_IO_ERROR)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = STATUS_OK; + +cleanup: + if (checked_clone_mini_live) { + platform_status cleanup_rc = blob_checksum_test_mini_deinit( + cc, &checked_clone_mini, checked_clone_meta_head); + if (SUCCESS(rc) && !SUCCESS(cleanup_rc)) { + rc = cleanup_rc; + } + } + if (source_mini_live) { + platform_status cleanup_rc = + blob_checksum_test_mini_deinit(cc, &source_mini, source_meta_head); + if (SUCCESS(rc) && !SUCCESS(cleanup_rc)) { + rc = cleanup_rc; + } + } + writable_buffer_deinit(&invalid_clone); + writable_buffer_deinit(&checked_clone); + writable_buffer_deinit(&materialized); + writable_buffer_deinit(&descriptor); + writable_buffer_deinit(&source_data); + + ASSERT_TRUE(SUCCESS(rc), + "blob checksum test failed: %s", + platform_status_to_string(rc)); +} + /* * ************************************************************************** * Basic test case to verify trunk_insert() API and validate a very large # @@ -302,13 +1177,1649 @@ CTEST2(splinter, test_two_log_checkpoint) merge_accumulator_to_message(lookup_result_accumulator(&qdata)), TRUE); } - lookup_result_deinit(&qdata); + lookup_result_deinit(&qdata); + + // Second checkpoint with no new inserts is a clean rotate. + rc = core_checkpoint(&spl, 0); + ASSERT_TRUE(SUCCESS(rc)); + + core_destroy(&spl); +} + +/* + * Pause an insert after it has reserved a log group and made its memtable + * mutation visible, but before it consumes the reservation. A durability + * barrier must cut that group and wait for the reserved write. It must not + * take insert exclusion: a second insert should reserve the new group and + * finish while both the first insert and the barrier remain blocked. + */ +CTEST2(splinter, test_durable_barrier_waits_for_visible_insert_log_write) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 0; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + DECLARE_AUTO_KEY_BUFFER(first_keybuf, data->hid); + DECLARE_AUTO_KEY_BUFFER(second_keybuf, data->hid); + merge_accumulator first_msg; + merge_accumulator second_msg; + merge_accumulator_init(&first_msg, data->hid); + merge_accumulator_init(&second_msg, data->hid); + test_key( + &first_keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 1, &first_msg); + test_key( + &second_keybuf, TEST_RANDOM, 2, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 2, &second_msg); + + core_durable_barrier_insert_args first_insert_args = { + .spl = &spl, + .tuple_key = key_buffer_key(&first_keybuf), + .msg = merge_accumulator_to_message(&first_msg), + .rc = STATUS_INVALID_STATE, + }; + core_durable_barrier_insert_args second_insert_args = { + .spl = &spl, + .tuple_key = key_buffer_key(&second_keybuf), + .msg = merge_accumulator_to_message(&second_msg), + .rc = STATUS_INVALID_STATE, + }; + core_durable_barrier_thread_args barrier_args = { + .spl = &spl, + .rc = STATUS_INVALID_STATE, + }; + platform_thread first_insert_thread = {0}; + platform_thread second_insert_thread = {0}; + platform_thread barrier_thread = {0}; + core_log_write_block log_block = {0}; + + lookup_result qdata; + lookup_result_init( + &qdata, spl.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + + bool32 first_insert_created = FALSE; + bool32 second_insert_created = FALSE; + bool32 barrier_created = FALSE; + bool32 write_blocked = FALSE; + bool32 visible_before_log_write = FALSE; + bool32 log_cut_while_writer_blocked = FALSE; + bool32 second_insert_completed = FALSE; + bool32 barrier_completed_before_release = FALSE; + platform_status lookup_rc = STATUS_INVALID_STATE; + platform_status first_insert_create_rc = STATUS_INVALID_STATE; + platform_status second_insert_create_rc = STATUS_INVALID_STATE; + platform_status barrier_create_rc = STATUS_INVALID_STATE; + platform_status first_insert_join_rc = STATUS_INVALID_STATE; + platform_status second_insert_join_rc = STATUS_INVALID_STATE; + platform_status barrier_join_rc = STATUS_INVALID_STATE; + + core_log_write_block_install(&log_block, spl.log); + first_insert_create_rc = + platform_thread_create(&first_insert_thread, + FALSE, + core_durable_barrier_insert_thread, + &first_insert_args, + data->hid); + first_insert_created = SUCCESS(first_insert_create_rc); + if (first_insert_created) { + write_blocked = core_durable_barrier_test_wait(&log_block.entered); + } + + if (write_blocked) { + lookup_rc = core_lookup(&spl, first_insert_args.tuple_key, &qdata); + if (SUCCESS(lookup_rc)) { + visible_before_log_write = + message_lex_cmp( + first_insert_args.msg, + merge_accumulator_to_message(lookup_result_accumulator(&qdata))) + == 0; + } + + barrier_create_rc = + platform_thread_create(&barrier_thread, + FALSE, + core_durable_barrier_test_thread, + &barrier_args, + data->hid); + barrier_created = SUCCESS(barrier_create_rc); + if (barrier_created) { + log_cut_while_writer_blocked = + core_durable_barrier_test_wait_for_live_log_cut( + (shard_log *)spl.log); + if (log_cut_while_writer_blocked) { + second_insert_create_rc = + platform_thread_create(&second_insert_thread, + FALSE, + core_durable_barrier_insert_thread, + &second_insert_args, + data->hid); + second_insert_created = SUCCESS(second_insert_create_rc); + if (second_insert_created) { + second_insert_completed = + core_durable_barrier_test_wait(&second_insert_args.done); + } + } + barrier_completed_before_release = + __atomic_load_n(&barrier_args.done, __ATOMIC_ACQUIRE); + } + } + + core_log_write_block_release(&log_block); + if (first_insert_created) { + first_insert_join_rc = platform_thread_join(&first_insert_thread); + } + if (second_insert_created) { + second_insert_join_rc = platform_thread_join(&second_insert_thread); + } + if (barrier_created) { + barrier_join_rc = platform_thread_join(&barrier_thread); + } + core_log_write_block_uninstall(&log_block); + + lookup_result_deinit(&qdata); + merge_accumulator_deinit(&second_msg); + merge_accumulator_deinit(&first_msg); + core_destroy(&spl); + + ASSERT_TRUE(SUCCESS(first_insert_create_rc)); + ASSERT_TRUE(write_blocked, + "insert did not reach the reserved-write blocking hook\n"); + ASSERT_TRUE(SUCCESS(lookup_rc), + "lookup of the paused insert failed: %s\n", + platform_status_to_string(lookup_rc)); + ASSERT_TRUE(visible_before_log_write, + "paused insert was not visible before its reserved write\n"); + ASSERT_TRUE(SUCCESS(barrier_create_rc)); + ASSERT_TRUE( + log_cut_while_writer_blocked, + "core_durable_barrier did not cut the reserved writer's group\n"); + ASSERT_TRUE(SUCCESS(second_insert_create_rc)); + ASSERT_TRUE( + second_insert_completed, + "core_durable_barrier excluded a writer after cutting the log\n"); + ASSERT_FALSE(barrier_completed_before_release, + "core_durable_barrier passed a visible reserved insert\n"); + ASSERT_TRUE(SUCCESS(first_insert_join_rc)); + ASSERT_TRUE(SUCCESS(second_insert_join_rc)); + ASSERT_TRUE(SUCCESS(barrier_join_rc)); + ASSERT_TRUE(SUCCESS(first_insert_args.rc), + "paused insert failed: %s\n", + platform_status_to_string(first_insert_args.rc)); + ASSERT_TRUE(SUCCESS(second_insert_args.rc), + "concurrent insert failed: %s\n", + platform_status_to_string(second_insert_args.rc)); + ASSERT_TRUE(SUCCESS(barrier_args.rc), + "core_durable_barrier failed: %s\n", + platform_status_to_string(barrier_args.rc)); +} + +/* + * Closing a group coalesces the small private tails left by concurrent writers + * into one page. That page also carries the group terminator, so the group's + * durable page count is one rather than one page per writer plus a dedicated + * terminator. Keep every worker alive until after inspection to prevent the + * platform from recycling thread IDs and accidentally sharing a tail buffer. + */ +#define CORE_DURABLE_BARRIER_TAIL_WRITERS 4 +CTEST2(splinter, test_durable_barrier_packs_concurrent_small_tails) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 0; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + key_buffer keybuf[CORE_DURABLE_BARRIER_TAIL_WRITERS]; + merge_accumulator msg[CORE_DURABLE_BARRIER_TAIL_WRITERS]; + core_durable_barrier_insert_args + insert_args[CORE_DURABLE_BARRIER_TAIL_WRITERS]; + platform_thread insert_thread[CORE_DURABLE_BARRIER_TAIL_WRITERS] = {0}; + platform_status join_rc[CORE_DURABLE_BARRIER_TAIL_WRITERS]; + bool32 start = FALSE; + bool32 release = FALSE; + + uint64 num_initialized = 0; + uint64 num_created = 0; + for (uint64 i = 0; i < CORE_DURABLE_BARRIER_TAIL_WRITERS; i++) { + key_buffer_init(&keybuf[i], data->hid); + merge_accumulator_init(&msg[i], data->hid); + num_initialized++; + test_key( + &keybuf[i], TEST_RANDOM, i + 1, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, i + 1, &msg[i]); + insert_args[i] = (core_durable_barrier_insert_args){ + .spl = &spl, + .tuple_key = key_buffer_key(&keybuf[i]), + .msg = merge_accumulator_to_message(&msg[i]), + .start = &start, + .release = &release, + .tid = INVALID_TID, + .rc = STATUS_INVALID_STATE, + }; + join_rc[i] = STATUS_INVALID_STATE; + + rc = platform_thread_create(&insert_thread[i], + FALSE, + core_durable_barrier_insert_thread, + &insert_args[i], + data->hid); + if (!SUCCESS(rc)) { + break; + } + num_created++; + } + + bool32 all_created = num_created == CORE_DURABLE_BARRIER_TAIL_WRITERS; + bool32 all_ready = all_created + && core_durable_barrier_test_wait_for_insert_threads( + insert_args, num_created, FALSE); + bool32 distinct_tids = all_ready; + if (all_ready) { + for (uint64 i = 0; i < num_created; i++) { + distinct_tids &= insert_args[i].tid < MAX_THREADS; + for (uint64 j = 0; j < i; j++) { + distinct_tids &= insert_args[i].tid != insert_args[j].tid; + } + } + } + + __atomic_store_n(&start, TRUE, __ATOMIC_RELEASE); + bool32 all_inserted = all_ready + && core_durable_barrier_test_wait_for_insert_threads( + insert_args, num_created, TRUE); + bool32 inserts_succeeded = all_inserted; + if (all_inserted) { + for (uint64 i = 0; i < num_created; i++) { + inserts_succeeded &= SUCCESS(insert_args[i].rc); + } + } + + core_durable_barrier_thread_args barrier_args = { + .spl = &spl, + .rc = STATUS_INVALID_STATE, + }; + platform_thread barrier_thread = {0}; + platform_status barrier_create_rc = STATUS_INVALID_STATE; + platform_status barrier_join_rc = STATUS_INVALID_STATE; + bool32 barrier_created = FALSE; + bool32 barrier_blocked = FALSE; + bool32 inspected_group = FALSE; + uint64 page_count = 0; + + if (inserts_succeeded && distinct_tids) { + checkpoint_barrier_fault_install(&data->checkpoint_fault, data->io); + checkpoint_barrier_fault_block(&data->checkpoint_fault, 0); + barrier_create_rc = + platform_thread_create(&barrier_thread, + FALSE, + core_durable_barrier_test_thread, + &barrier_args, + data->hid); + barrier_created = SUCCESS(barrier_create_rc); + if (barrier_created) { + barrier_blocked = core_durable_barrier_test_wait( + &data->checkpoint_fault.block_entered); + } + + if (barrier_blocked) { + shard_log *log = (shard_log *)spl.log; + platform_mutex_lock(&log->graduate_lock); + shard_log_group *closed = log->groups_head; + shard_log_group *accepting = + __atomic_load_n(&log->accepting.group, __ATOMIC_SEQ_CST); + inspected_group = closed != NULL && closed != accepting; + if (inspected_group) { + page_count = closed->page_count; + } + platform_mutex_unlock(&log->graduate_lock); + } + + checkpoint_barrier_fault_release(&data->checkpoint_fault); + if (barrier_created) { + barrier_join_rc = platform_thread_join(&barrier_thread); + } + checkpoint_barrier_fault_uninstall(&data->checkpoint_fault); + } + + __atomic_store_n(&release, TRUE, __ATOMIC_RELEASE); + for (uint64 i = 0; i < num_created; i++) { + join_rc[i] = platform_thread_join(&insert_thread[i]); + } + for (uint64 i = 0; i < num_initialized; i++) { + merge_accumulator_deinit(&msg[i]); + key_buffer_deinit(&keybuf[i]); + } + core_destroy(&spl); + + ASSERT_TRUE(all_created, "failed to create all concurrent log writers\n"); + ASSERT_TRUE(all_ready, "concurrent log writers did not reach their gate\n"); + ASSERT_TRUE(distinct_tids, + "concurrent log writers did not retain distinct thread IDs\n"); + ASSERT_TRUE(all_inserted, "concurrent log writers did not finish inserts\n"); + ASSERT_TRUE(inserts_succeeded, "a concurrent log writer failed\n"); + ASSERT_TRUE(SUCCESS(barrier_create_rc)); + ASSERT_TRUE(barrier_blocked, + "core_durable_barrier did not reach the device barrier\n"); + ASSERT_TRUE(inspected_group, + "could not inspect the closed durability group\n"); + ASSERT_EQUAL(1, + page_count, + "small concurrent tails used %lu log pages instead of one\n", + page_count); + ASSERT_TRUE(SUCCESS(barrier_join_rc)); + ASSERT_TRUE(SUCCESS(barrier_args.rc), + "core_durable_barrier failed: %s\n", + platform_status_to_string(barrier_args.rc)); + for (uint64 i = 0; i < num_created; i++) { + ASSERT_TRUE(SUCCESS(join_rc[i])); + } +} +#undef CORE_DURABLE_BARRIER_TAIL_WRITERS + +/* + * This order is a counterexample for both the old in-thread-order First Fit + * packer and a largest-first packer which fills bins from the smallest item + * backward. With C as the page payload capacity, those algorithms produce + * three pages: + * + * old FF: (.08 + .18 + .18), .68, .78 + * reverse: (.78 + .08), (.68 + .18), .18 + * + * First Fit Decreasing instead produces exactly two: + * + * (.78 + .18), (.68 + .18 + .08) + */ +#define SHARD_LOG_FFD_TEST_WRITERS 5 +static const uint64 shard_log_ffd_test_percent[SHARD_LOG_FFD_TEST_WRITERS] = { + 8, + 18, + 18, + 68, + 78, +}; + +typedef struct shard_log_ffd_test_writer { + log_handle *log; + platform_thread thread; + bool32 *start; + bool32 *release; + message msg; + uint64 entry_num; + uint64 key_size; + threadid tid; + int log_rc; + bool32 ready; + bool32 done; +} shard_log_ffd_test_writer; + +static void +shard_log_ffd_test_write(void *arg) +{ + shard_log_ffd_test_writer *writer = arg; + platform_heap_id hid = platform_get_heap_id(); + DECLARE_AUTO_KEY_BUFFER(keybuf, hid); + + writer->tid = platform_get_tid(); + __atomic_store_n(&writer->ready, TRUE, __ATOMIC_RELEASE); + while (!__atomic_load_n(writer->start, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(USEC_TO_NSEC(50)); + } + + key tuple_key = test_key( + &keybuf, TEST_RANDOM, writer->entry_num, 0, 0, writer->key_size, 0); + writer->log_rc = + log_write(writer->log, tuple_key, writer->msg, writer->entry_num, 0); + __atomic_store_n(&writer->done, TRUE, __ATOMIC_RELEASE); + + /* Keep the thread ID, and therefore its staging-buffer assignment, live. */ + while (!__atomic_load_n(writer->release, __ATOMIC_ACQUIRE)) { + platform_sleep_ns(USEC_TO_NSEC(50)); + } +} + +static bool32 +shard_log_ffd_test_wait_for_writers(shard_log_ffd_test_writer *writers, + bool32 wait_for_done) +{ + timestamp start = platform_get_timestamp(); + while (platform_timestamp_elapsed(start) + < CORE_DURABLE_BARRIER_TEST_TIMEOUT_NS) + { + bool32 all_reached = TRUE; + for (uint64 i = 0; i < SHARD_LOG_FFD_TEST_WRITERS; i++) { + const bool32 *flag = + wait_for_done ? &writers[i].done : &writers[i].ready; + all_reached &= __atomic_load_n(flag, __ATOMIC_ACQUIRE); + } + if (all_reached) { + return TRUE; + } + platform_sleep_ns(USEC_TO_NSEC(50)); + } + return FALSE; +} + +typedef struct shard_log_ffd_test_sealer { + log_handle *log; + platform_thread thread; + platform_status rc; +} shard_log_ffd_test_sealer; + +static void +shard_log_ffd_test_seal(void *arg) +{ + shard_log_ffd_test_sealer *sealer = arg; + sealer->rc = log_seal(sealer->log); +} + +CTEST2(splinter, test_shard_log_first_fit_decreasing_packing) +{ + const uint64 key_size = 8; + uint64 page_capacity = + cache_page_size((cache *)data->clock_cache) - sizeof(shard_log_hdr); + /* sizeof(log_entry), which is private to shard_log.c. */ + uint64 entry_overhead = 2 * sizeof(uint64) + sizeof(ondisk_tuple) + key_size; + + char *payload[SHARD_LOG_FFD_TEST_WRITERS] = {0}; + message expected_msg[SHARD_LOG_FFD_TEST_WRITERS]; + uint64 expected_size[SHARD_LOG_FFD_TEST_WRITERS]; + for (uint64 rank = 0; rank < SHARD_LOG_FFD_TEST_WRITERS; rank++) { + expected_size[rank] = + page_capacity * shard_log_ffd_test_percent[rank] / 100; + platform_assert(expected_size[rank] > entry_overhead); + uint64 message_size = expected_size[rank] - entry_overhead; + platform_assert(message_size <= UINT16_MAX); + payload[rank] = + TYPED_ARRAY_MALLOC(data->hid, payload[rank], message_size); + platform_assert(payload[rank] != NULL); + memset(payload[rank], (int)(rank + 1), message_size); + expected_msg[rank] = message_create( + MESSAGE_TYPE_INSERT, NULL, slice_create(message_size, payload[rank])); + } + + log_handle *log = NULL; + platform_assert_status_ok(shard_log_create( + (cache *)data->clock_cache, &data->system_cfg->log_cfg, data->hid, &log)); + log_head segment = log_get_head(log); + + shard_log_ffd_test_writer writers[SHARD_LOG_FFD_TEST_WRITERS] = {0}; + uint64 order[SHARD_LOG_FFD_TEST_WRITERS]; + bool32 selected[SHARD_LOG_FFD_TEST_WRITERS] = {0}; + bool32 start = FALSE; + bool32 release = FALSE; + for (uint64 i = 0; i < SHARD_LOG_FFD_TEST_WRITERS; i++) { + writers[i] = (shard_log_ffd_test_writer){ + .log = log, + .start = &start, + .release = &release, + .tid = INVALID_TID, + .log_rc = -1, + .key_size = key_size, + }; + platform_assert_status_ok(platform_thread_create(&writers[i].thread, + FALSE, + shard_log_ffd_test_write, + &writers[i], + data->hid)); + } + + bool32 all_ready = shard_log_ffd_test_wait_for_writers(writers, FALSE); + platform_assert(all_ready, "FFD test writers did not become ready"); + + /* Assign payload sizes by actual tid, not scheduler-dependent spawn order. + */ + for (uint64 rank = 0; rank < SHARD_LOG_FFD_TEST_WRITERS; rank++) { + threadid lowest_tid = INVALID_TID; + uint64 lowest_i = SHARD_LOG_FFD_TEST_WRITERS; + for (uint64 i = 0; i < SHARD_LOG_FFD_TEST_WRITERS; i++) { + if (!selected[i] && writers[i].tid < lowest_tid) { + lowest_tid = writers[i].tid; + lowest_i = i; + } + } + platform_assert(lowest_i != SHARD_LOG_FFD_TEST_WRITERS); + platform_assert(lowest_tid != 0, + "thread 0 must remain the empty final-page buffer"); + selected[lowest_i] = TRUE; + order[rank] = lowest_i; + writers[lowest_i].msg = expected_msg[rank]; + writers[lowest_i].entry_num = rank; + } + + __atomic_store_n(&start, TRUE, __ATOMIC_RELEASE); + bool32 all_done = shard_log_ffd_test_wait_for_writers(writers, TRUE); + platform_assert(all_done, "FFD test writers did not finish appending"); + for (uint64 i = 0; i < SHARD_LOG_FFD_TEST_WRITERS; i++) { + platform_assert(writers[i].log_rc == 0); + } + + shard_log *slog = (shard_log *)log; + shard_log_group *group = + __atomic_load_n(&slog->accepting.group, __ATOMIC_SEQ_CST); + platform_assert(group != NULL); + platform_assert(group->thread_data[0].offset == sizeof(shard_log_hdr)); + for (uint64 rank = 0; rank < SHARD_LOG_FFD_TEST_WRITERS; rank++) { + shard_log_ffd_test_writer *writer = &writers[order[rank]]; + shard_log_thread_data *thread_data = &group->thread_data[writer->tid]; + platform_assert(thread_data->state == SHARD_LOG_BUFFER_OPEN); + ASSERT_EQUAL(expected_size[rank], + thread_data->offset - sizeof(shard_log_hdr), + "writer rank %lu staged an unexpected payload size\n", + rank); + } + + checkpoint_barrier_fault_install(&data->checkpoint_fault, data->io); + checkpoint_barrier_fault_block(&data->checkpoint_fault, 0); + shard_log_ffd_test_sealer sealer = { + .log = log, + .rc = STATUS_INVALID_STATE, + }; + platform_assert_status_ok(platform_thread_create( + &sealer.thread, FALSE, shard_log_ffd_test_seal, &sealer, data->hid)); + bool32 barrier_blocked = + core_durable_barrier_test_wait(&data->checkpoint_fault.block_entered); + + uint64 page_count = 0; + bool32 inspected_group = FALSE; + if (barrier_blocked) { + platform_mutex_lock(&slog->graduate_lock); + shard_log_group *sealed_group = slog->groups_head; + shard_log_group *accepting = + __atomic_load_n(&slog->accepting.group, __ATOMIC_SEQ_CST); + inspected_group = sealed_group != NULL && accepting == NULL; + if (inspected_group) { + page_count = sealed_group->page_count; + } + platform_mutex_unlock(&slog->graduate_lock); + } + + checkpoint_barrier_fault_release(&data->checkpoint_fault); + platform_status seal_join_rc = platform_thread_join(&sealer.thread); + checkpoint_barrier_fault_uninstall(&data->checkpoint_fault); + + __atomic_store_n(&release, TRUE, __ATOMIC_RELEASE); + for (uint64 i = 0; i < SHARD_LOG_FFD_TEST_WRITERS; i++) { + platform_assert_status_ok(platform_thread_join(&writers[i].thread)); + } + + ASSERT_TRUE(barrier_blocked, + "sealed log did not reach the device barrier\n"); + ASSERT_TRUE(inspected_group, + "could not inspect the sealed FFD test group\n"); + ASSERT_EQUAL(2, + page_count, + "FFD packed the counterexample into %lu pages, not two\n", + page_count); + ASSERT_TRUE(SUCCESS(seal_join_rc)); + ASSERT_TRUE(SUCCESS(sealer.rc), + "sealing the FFD test log failed: %s\n", + platform_status_to_string(sealer.rc)); + + log_deinit(log); + + log_iterator *itor = NULL; + platform_status rc = shard_log_iterator_create((cache *)data->clock_cache, + &data->system_cfg->log_cfg, + data->hid, + segment, + 0, + &itor); + platform_assert_status_ok(rc); + ASSERT_TRUE(log_iterator_stream_complete(itor)); + DECLARE_AUTO_KEY_BUFFER(expected_keybuf, data->hid); + for (uint64 rank = 0; rank < SHARD_LOG_FFD_TEST_WRITERS; rank++) { + ASSERT_TRUE(log_iterator_can_next(itor)); + key expected_key = + test_key(&expected_keybuf, TEST_RANDOM, rank, 0, 0, key_size, 0); + key actual_key; + message actual_msg; + log_iterator_curr(itor, &actual_key, &actual_msg); + uint64 memtable_generation; + uint64 leaf_generation; + log_iterator_curr_generations( + itor, &memtable_generation, &leaf_generation); + ASSERT_EQUAL(rank, memtable_generation); + ASSERT_EQUAL(0, leaf_generation); + ASSERT_EQUAL(0, + data_key_compare( + data->system_cfg->data_cfg, expected_key, actual_key)); + ASSERT_EQUAL(0, message_lex_cmp(expected_msg[rank], actual_msg)); + platform_assert_status_ok(log_iterator_next(itor)); + } + ASSERT_FALSE(log_iterator_can_next(itor)); + log_iterator_deinit(itor); + shard_log_dec_ref((cache *)data->clock_cache, &segment); + + for (uint64 rank = 0; rank < SHARD_LOG_FFD_TEST_WRITERS; rank++) { + platform_free(data->hid, payload[rank]); + } +} +#undef SHARD_LOG_FFD_TEST_WRITERS + +typedef struct shard_log_page_alloc_fault { + cache *cc; + const cache_ops *saved_ops; + cache_ops fault_ops; + bool32 fail_next_log_page; + uint64 failures; +} shard_log_page_alloc_fault; + +static shard_log_page_alloc_fault *active_shard_log_page_alloc_fault; + +static page_handle * +shard_log_test_page_alloc(cache *cc, uint64 addr, page_type type) +{ + shard_log_page_alloc_fault *fault = + __atomic_load_n(&active_shard_log_page_alloc_fault, __ATOMIC_ACQUIRE); + platform_assert(fault != NULL && fault->cc == cc); + + if (type == PAGE_TYPE_LOG + && __atomic_exchange_n( + &fault->fail_next_log_page, FALSE, __ATOMIC_ACQ_REL)) + { + __atomic_fetch_add(&fault->failures, 1, __ATOMIC_RELAXED); + return NULL; + } + return fault->saved_ops->page_alloc(cc, addr, type); +} + +static void +shard_log_page_alloc_fault_install(shard_log_page_alloc_fault *fault, cache *cc) +{ + platform_assert( + __atomic_load_n(&active_shard_log_page_alloc_fault, __ATOMIC_ACQUIRE) + == NULL); + ZERO_CONTENTS(fault); + fault->cc = cc; + fault->saved_ops = cc->ops; + fault->fault_ops = *cc->ops; + fault->fault_ops.page_alloc = shard_log_test_page_alloc; + fault->fail_next_log_page = TRUE; + __atomic_store_n( + &active_shard_log_page_alloc_fault, fault, __ATOMIC_RELEASE); + cc->ops = &fault->fault_ops; +} + +static void +shard_log_page_alloc_fault_uninstall(shard_log_page_alloc_fault *fault) +{ + platform_assert( + __atomic_load_n(&active_shard_log_page_alloc_fault, __ATOMIC_ACQUIRE) + == fault); + fault->cc->ops = fault->saved_ops; + __atomic_store_n(&active_shard_log_page_alloc_fault, NULL, __ATOMIC_RELEASE); + fault->cc = NULL; + fault->saved_ops = NULL; +} + +/* + * Page-slot allocation happens before an OPEN staging image is frozen. A + * transient failure must therefore leave the image mutable and intact; a + * later seal can allocate a different page and finish the exact same record. + */ +CTEST2(splinter, test_shard_log_page_alloc_failure_retries_open_buffer) +{ + cache *cc = (cache *)data->clock_cache; + log_handle *log; + platform_assert_status_ok( + shard_log_create(cc, &data->system_cfg->log_cfg, data->hid, &log)); + log_head segment = log_get_head(log); + + const uint64 entry_num = 8675309; + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + generate_test_message(&data->gen, entry_num, &msg); + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + key tuple_key = test_key( + &keybuf, TEST_RANDOM, entry_num, 0, 0, data->workload_cfg->key_size, 0); + int log_rc = log_write( + log, tuple_key, merge_accumulator_to_message(&msg), entry_num, 0); + platform_assert(log_rc == 0); + + shard_log_page_alloc_fault fault; + shard_log_page_alloc_fault_install(&fault, cc); + platform_status first_seal_rc = log_seal(log); + shard_log_page_alloc_fault_uninstall(&fault); + + ASSERT_TRUE(STATUS_IS_EQ(first_seal_rc, STATUS_NO_SPACE), + "faulted seal returned %s, not out-of-space\n", + platform_status_to_string(first_seal_rc)); + ASSERT_EQUAL(1, __atomic_load_n(&fault.failures, __ATOMIC_RELAXED)); + + shard_log *slog = (shard_log *)log; + platform_mutex_lock(&slog->graduate_lock); + shard_log_group *group = slog->groups_head; + bool32 group_is_retryable = FALSE; + if (group != NULL) { + shard_log_thread_data *final = &group->thread_data[0]; + uint64 writeback_requests = 0; + for (threadid tid = 0; tid < MAX_THREADS; tid++) { + writeback_requests += + writeback_set_num_requests(&group->thread_data[tid].wbset); + } + group_is_retryable = group->state == SHARD_LOG_GROUP_TERMINATING + && final->state == SHARD_LOG_BUFFER_OPEN + && final->offset > sizeof(shard_log_hdr) + && final->incache_page == NULL + && group->page_count == 0 && writeback_requests == 0; + } + platform_mutex_unlock(&slog->graduate_lock); + ASSERT_TRUE(group_is_retryable, + "page allocation failure did not preserve the OPEN image\n"); + + platform_status retry_rc = log_seal(log); + ASSERT_TRUE(SUCCESS(retry_rc), + "seal retry failed: %s\n", + platform_status_to_string(retry_rc)); + log_deinit(log); + + log_iterator *itor = NULL; + platform_status rc = shard_log_iterator_create( + cc, &data->system_cfg->log_cfg, data->hid, segment, 0, &itor); + platform_assert_status_ok(rc); + ASSERT_TRUE(log_iterator_stream_complete(itor)); + ASSERT_TRUE(log_iterator_can_next(itor)); + + key replayed_key; + message replayed_msg; + log_iterator_curr(itor, &replayed_key, &replayed_msg); + uint64 memtable_generation; + uint64 leaf_generation; + log_iterator_curr_generations(itor, &memtable_generation, &leaf_generation); + ASSERT_EQUAL(entry_num, memtable_generation); + ASSERT_EQUAL(0, leaf_generation); + ASSERT_EQUAL( + 0, data_key_compare(data->system_cfg->data_cfg, tuple_key, replayed_key)); + ASSERT_EQUAL( + 0, message_lex_cmp(merge_accumulator_to_message(&msg), replayed_msg)); + platform_assert_status_ok(log_iterator_next(itor)); + ASSERT_FALSE(log_iterator_can_next(itor)); + + log_iterator_deinit(itor); + shard_log_dec_ref(cc, &segment); + merge_accumulator_deinit(&msg); +} + +/* Without a WAL, the barrier must fold its frontier into a durable COW root. */ +CTEST2(splinter, test_durable_barrier_without_log_publishes_root) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = FALSE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 0; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_NULL(spl.log); + + uint64 insert_generation = memtable_generation(&spl.mt_ctxt); + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + test_key(&keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 1, &msg); + rc = core_insert( + &spl, key_buffer_key(&keybuf), merge_accumulator_to_message(&msg), NULL); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_durable_barrier(&spl); + ASSERT_TRUE(SUCCESS(rc), + "no-log core_durable_barrier failed: %s\n", + platform_status_to_string(rc)); + + /* Read a fresh image from disk rather than trusting the live context. */ + superblock_context disk_superblock; + allocator_config *allocator_cfg = allocator_get_config(alp); + rc = superblock_context_init( + &disk_superblock, data->io, allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&disk_superblock, allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + + superblock_tree_record record; + superblock_get_tree_record(&disk_superblock, &record); + ASSERT_NOT_EQUAL(0, record.root_addr); + ASSERT_TRUE(record.first_unincorporated_generation > insert_generation, + "durable root stops at generation %lu, insert was in %lu\n", + record.first_unincorporated_generation, + insert_generation); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(record.sealed_log)); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(record.live_log)); + + superblock_context_deinit(&disk_superblock); + merge_accumulator_deinit(&msg); + core_destroy(&spl); +} + +/* + * The durability cut may briefly exclude inserts, but the slow device barrier + * must not. Hold the first device barrier after a logged update reaches it, + * then require a new insert to finish while the barrier thread is still held + * inside the I/O hook. Releasing the hook must let the original durability + * call finish successfully. + * + * No CTest assertion is made while the hook is installed or either worker may + * still be live. That keeps a failed assertion from stranding a registered + * thread in the blocking hook and makes fixture cleanup deterministic. + */ +CTEST2(splinter, test_durable_barrier_reopens_writers_before_device_barrier) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 0; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + + test_key(&keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 1, &msg); + rc = core_insert( + &spl, key_buffer_key(&keybuf), merge_accumulator_to_message(&msg), NULL); + ASSERT_TRUE(SUCCESS(rc)); + rc = task_perform_until_quiescent(spl.ts); + ASSERT_TRUE(SUCCESS(rc)); + + /* Keep the second tuple's storage alive until its worker has joined. */ + test_key(&keybuf, TEST_RANDOM, 2, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 2, &msg); + core_durable_barrier_insert_args insert_args = { + .spl = &spl, + .tuple_key = key_buffer_key(&keybuf), + .msg = merge_accumulator_to_message(&msg), + .rc = STATUS_INVALID_STATE, + }; + core_durable_barrier_thread_args barrier_args = { + .spl = &spl, + .rc = STATUS_INVALID_STATE, + }; + + platform_thread barrier_thread = {0}; + platform_thread insert_thread = {0}; + bool32 barrier_created = FALSE; + bool32 insert_created = FALSE; + bool32 barrier_blocked = FALSE; + bool32 insert_completed_before_release = FALSE; + bool32 barrier_completed_before_release = FALSE; + platform_status barrier_create_rc = STATUS_INVALID_STATE; + platform_status insert_create_rc = STATUS_INVALID_STATE; + platform_status barrier_join_rc = STATUS_INVALID_STATE; + platform_status insert_join_rc = STATUS_INVALID_STATE; + + checkpoint_barrier_fault_install(&data->checkpoint_fault, data->io); + checkpoint_barrier_fault_block(&data->checkpoint_fault, 0); + + barrier_create_rc = platform_thread_create(&barrier_thread, + FALSE, + core_durable_barrier_test_thread, + &barrier_args, + data->hid); + barrier_created = SUCCESS(barrier_create_rc); + if (barrier_created) { + barrier_blocked = + core_durable_barrier_test_wait(&data->checkpoint_fault.block_entered); + } + + if (barrier_blocked) { + insert_create_rc = + platform_thread_create(&insert_thread, + FALSE, + core_durable_barrier_insert_thread, + &insert_args, + data->hid); + insert_created = SUCCESS(insert_create_rc); + if (insert_created) { + insert_completed_before_release = + core_durable_barrier_test_wait(&insert_args.done); + } + barrier_completed_before_release = + __atomic_load_n(&barrier_args.done, __ATOMIC_ACQUIRE); + } + + checkpoint_barrier_fault_release(&data->checkpoint_fault); + if (insert_created) { + insert_join_rc = platform_thread_join(&insert_thread); + } + if (barrier_created) { + barrier_join_rc = platform_thread_join(&barrier_thread); + } + checkpoint_barrier_fault_uninstall(&data->checkpoint_fault); + + merge_accumulator_deinit(&msg); + core_destroy(&spl); + + ASSERT_TRUE(SUCCESS(barrier_create_rc)); + ASSERT_TRUE(barrier_blocked, + "core_durable_barrier did not reach the device barrier\n"); + ASSERT_TRUE(SUCCESS(insert_create_rc)); + ASSERT_TRUE(insert_completed_before_release, + "insert did not finish while the device barrier was blocked\n"); + ASSERT_FALSE(barrier_completed_before_release, + "core_durable_barrier returned before its device barrier\n"); + ASSERT_TRUE(SUCCESS(insert_join_rc)); + ASSERT_TRUE(SUCCESS(barrier_join_rc)); + ASSERT_TRUE(SUCCESS(insert_args.rc), + "concurrent insert failed: %s\n", + platform_status_to_string(insert_args.rc)); + ASSERT_TRUE(SUCCESS(barrier_args.rc), + "core_durable_barrier failed: %s\n", + platform_status_to_string(barrier_args.rc)); +} + +/* + * Two barriers over the same write frontier share one group ticket. Hold the + * first caller at the device, wait until both begin calls have pinned their + * tickets, then release them together. The second caller must neither return + * early nor issue a redundant device barrier. + */ +CTEST2(splinter, test_concurrent_durable_barriers_coalesce) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 0; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + test_key(&keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 1, &msg); + rc = core_insert( + &spl, key_buffer_key(&keybuf), merge_accumulator_to_message(&msg), NULL); + ASSERT_TRUE(SUCCESS(rc)); + rc = task_perform_until_quiescent(spl.ts); + ASSERT_TRUE(SUCCESS(rc)); + + core_durable_barrier_thread_args first = { + .spl = &spl, + .rc = STATUS_INVALID_STATE, + }; + core_durable_barrier_thread_args second = { + .spl = &spl, + .rc = STATUS_INVALID_STATE, + }; + platform_thread first_thread = {0}; + platform_thread second_thread = {0}; + bool32 first_created = FALSE; + bool32 second_created = FALSE; + bool32 first_blocked = FALSE; + bool32 both_tickets_pinned = FALSE; + bool32 second_completed_before_release = FALSE; + platform_status first_create_rc = STATUS_INVALID_STATE; + platform_status second_create_rc = STATUS_INVALID_STATE; + platform_status first_join_rc = STATUS_INVALID_STATE; + platform_status second_join_rc = STATUS_INVALID_STATE; + + checkpoint_barrier_fault_install(&data->checkpoint_fault, data->io); + checkpoint_barrier_fault_block(&data->checkpoint_fault, 0); + + first_create_rc = platform_thread_create(&first_thread, + FALSE, + core_durable_barrier_test_thread, + &first, + data->hid); + first_created = SUCCESS(first_create_rc); + if (first_created) { + first_blocked = + core_durable_barrier_test_wait(&data->checkpoint_fault.block_entered); + } + + if (first_blocked) { + second_create_rc = + platform_thread_create(&second_thread, + FALSE, + core_durable_barrier_test_thread, + &second, + data->hid); + second_created = SUCCESS(second_create_rc); + if (second_created) { + both_tickets_pinned = core_durable_barrier_test_wait_for_handle_refs( + (shard_log *)spl.log, 3); + second_completed_before_release = + __atomic_load_n(&second.done, __ATOMIC_ACQUIRE); + } + } + + checkpoint_barrier_fault_release(&data->checkpoint_fault); + if (second_created) { + second_join_rc = platform_thread_join(&second_thread); + } + if (first_created) { + first_join_rc = platform_thread_join(&first_thread); + } + uint64 device_barriers = + __atomic_load_n(&data->checkpoint_fault.barriers, __ATOMIC_ACQUIRE); + checkpoint_barrier_fault_uninstall(&data->checkpoint_fault); + + merge_accumulator_deinit(&msg); + core_destroy(&spl); + + ASSERT_TRUE(SUCCESS(first_create_rc)); + ASSERT_TRUE(first_blocked, + "first core_durable_barrier did not reach the device\n"); + ASSERT_TRUE(SUCCESS(second_create_rc)); + ASSERT_TRUE(both_tickets_pinned, + "concurrent barriers did not both pin the in-flight ticket\n"); + ASSERT_FALSE(second_completed_before_release, + "second barrier passed an undurable shared ticket\n"); + ASSERT_TRUE(SUCCESS(first_join_rc)); + ASSERT_TRUE(SUCCESS(second_join_rc)); + ASSERT_TRUE(SUCCESS(first.rc), + "first core_durable_barrier failed: %s\n", + platform_status_to_string(first.rc)); + ASSERT_TRUE(SUCCESS(second.rc), + "second core_durable_barrier failed: %s\n", + platform_status_to_string(second.rc)); + ASSERT_EQUAL(1, + device_barriers, + "coalesced barriers issued %lu device barriers\n", + device_barriers); +} + +/* + * A checkpoint cut is not recoverable through its new live log until the + * superblock durably names both the retired and live streams. Block exactly + * that publication barrier (the retired-log seal is the preceding barrier), + * append to the already-installed live log, and start a durability barrier. + * The live log may become durable independently, but the core barrier must not + * return until the blocked cut publication completes. + * + * As in the other blocking-hook tests, collect predicates while workers are + * live and make CTest assertions only after releasing the hook and joining + * both workers. + */ +CTEST2(splinter, test_durable_barrier_waits_for_checkpoint_publication) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 0; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + test_key(&keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 1, &msg); + rc = core_insert( + &spl, key_buffer_key(&keybuf), merge_accumulator_to_message(&msg), NULL); + ASSERT_TRUE(SUCCESS(rc)); + + platform_mutex_lock(&spl.checkpoint_state_lock); + uint64 publication_target = spl.checkpoint.publications + 1; + platform_mutex_unlock(&spl.checkpoint_state_lock); + + core_checkpoint_thread_args checkpoint_args = { + .spl = &spl, + .rc = STATUS_INVALID_STATE, + }; + core_durable_barrier_thread_args barrier_args = { + .spl = &spl, + .rc = STATUS_INVALID_STATE, + }; + platform_thread checkpoint_thread = {0}; + platform_thread barrier_thread = {0}; + + bool32 checkpoint_created = FALSE; + bool32 barrier_created = FALSE; + bool32 publication_blocked = FALSE; + bool32 observed_publishing = FALSE; + bool32 retired_log_already_sealed = FALSE; + bool32 live_insert_succeeded = FALSE; + bool32 live_barrier_reached_device = FALSE; + bool32 live_log_durable_before_publication = FALSE; + bool32 barrier_completed_before_release = FALSE; + bool32 checkpoint_completed_before_release = FALSE; + bool32 publication_completed = FALSE; + platform_status checkpoint_create_rc = STATUS_INVALID_STATE; + platform_status barrier_create_rc = STATUS_INVALID_STATE; + platform_status live_insert_rc = STATUS_INVALID_STATE; + platform_status checkpoint_join_rc = STATUS_INVALID_STATE; + platform_status barrier_join_rc = STATUS_INVALID_STATE; + + checkpoint_barrier_fault_install(&data->checkpoint_fault, data->io); + /* Seal is barrier 0; cut publication is barrier 1. */ + checkpoint_barrier_fault_block(&data->checkpoint_fault, 1); + + checkpoint_create_rc = platform_thread_create(&checkpoint_thread, + FALSE, + core_checkpoint_test_thread, + &checkpoint_args, + data->hid); + checkpoint_created = SUCCESS(checkpoint_create_rc); + if (checkpoint_created) { + publication_blocked = + core_durable_barrier_test_wait(&data->checkpoint_fault.block_entered); + } + + if (publication_blocked) { + platform_mutex_lock(&spl.checkpoint_state_lock); + observed_publishing = spl.checkpoint.phase == CORE_CHECKPOINT_PUBLISHING; + retired_log_already_sealed = spl.checkpoint.log_to_seal == NULL; + platform_mutex_unlock(&spl.checkpoint_state_lock); + checkpoint_completed_before_release = + __atomic_load_n(&checkpoint_args.done, __ATOMIC_ACQUIRE); + + /* This update belongs to the new live log named by the blocked cut. */ + test_key(&keybuf, TEST_RANDOM, 2, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 2, &msg); + live_insert_rc = core_insert(&spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + live_insert_succeeded = SUCCESS(live_insert_rc); + } + + if (live_insert_succeeded) { + barrier_create_rc = + platform_thread_create(&barrier_thread, + FALSE, + core_durable_barrier_test_thread, + &barrier_args, + data->hid); + barrier_created = SUCCESS(barrier_create_rc); + if (barrier_created) { + /* Barriers 0 and 1 belong to the checkpoint; 2 is the live log. */ + live_barrier_reached_device = + core_durable_barrier_test_wait_for_io_barriers( + &data->checkpoint_fault, 3); + if (live_barrier_reached_device) { + live_log_durable_before_publication = + core_durable_barrier_test_wait_for_live_log_durable( + (shard_log *)spl.log); + } + barrier_completed_before_release = + __atomic_load_n(&barrier_args.done, __ATOMIC_ACQUIRE); + } + } + + checkpoint_barrier_fault_release(&data->checkpoint_fault); + if (barrier_created) { + barrier_join_rc = platform_thread_join(&barrier_thread); + } + if (checkpoint_created) { + checkpoint_join_rc = platform_thread_join(&checkpoint_thread); + } + platform_mutex_lock(&spl.checkpoint_state_lock); + publication_completed = spl.checkpoint.publications >= publication_target; + platform_mutex_unlock(&spl.checkpoint_state_lock); + checkpoint_barrier_fault_uninstall(&data->checkpoint_fault); + + merge_accumulator_deinit(&msg); + core_destroy(&spl); + + ASSERT_TRUE(SUCCESS(checkpoint_create_rc)); + ASSERT_TRUE(publication_blocked, + "checkpoint did not reach its cut-publication barrier\n"); + ASSERT_TRUE(observed_publishing, + "checkpoint was not PUBLISHING at the blocked barrier\n"); + ASSERT_TRUE(retired_log_already_sealed, + "blocked the retired-log seal rather than cut publication\n"); + ASSERT_FALSE(checkpoint_completed_before_release, + "checkpoint returned before its publication barrier\n"); + ASSERT_TRUE(live_insert_succeeded, + "insert into the new live log failed: %s\n", + platform_status_to_string(live_insert_rc)); + ASSERT_TRUE(SUCCESS(barrier_create_rc)); + ASSERT_TRUE(live_barrier_reached_device, + "core_durable_barrier did not reach the live-log barrier\n"); + ASSERT_TRUE(live_log_durable_before_publication, + "the new live log did not become durable while publication " + "was blocked\n"); + ASSERT_FALSE(barrier_completed_before_release, + "core_durable_barrier returned before cut publication\n"); + ASSERT_TRUE(SUCCESS(barrier_join_rc)); + ASSERT_TRUE(SUCCESS(checkpoint_join_rc)); + ASSERT_TRUE(SUCCESS(barrier_args.rc), + "core_durable_barrier failed: %s\n", + platform_status_to_string(barrier_args.rc)); + ASSERT_TRUE(SUCCESS(checkpoint_args.rc), + "core_checkpoint failed: %s\n", + platform_status_to_string(checkpoint_args.rc)); + ASSERT_TRUE(publication_completed, + "checkpoint publication counter did not advance\n"); +} + +/* + * A failed cut publication leaves the sealed retiring log durably named and + * the empty replacement log unnamed. If that replacement's first append + * fails before accepting a record, log_is_empty() remains true even though the + * memtable mutation is visible. With root publication faulted as well, + * unmount must reject the retiring-log fallback because make_durable() exposes + * the replacement log's poisoned group. + */ +CTEST2(splinter, test_unmount_rejects_empty_poisoned_replacement_log) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 0; + + allocator_root_id root_id = test_generate_allocator_root_id(); + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + test_key(&keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 1, &msg); + rc = core_insert( + &spl, key_buffer_key(&keybuf), merge_accumulator_to_message(&msg), NULL); + ASSERT_TRUE(SUCCESS(rc)); + + log_head retiring_log = log_get_head(spl.log); + checkpoint_barrier_fault_install(&data->checkpoint_fault, data->io); + /* Barrier 0 seals retiring_log; barrier 1 faults cut publication. */ + checkpoint_barrier_fault_arm(&data->checkpoint_fault, 1); + platform_status checkpoint_rc = core_checkpoint(&spl, 0); + platform_status checkpoint_quiesce_rc = task_perform_until_quiescent(spl.ts); + + platform_mutex_lock(&spl.checkpoint_state_lock); + bool32 unpublished_sealed_cut = + spl.checkpoint.phase == CORE_CHECKPOINT_SEALING + && spl.checkpoint.log_to_seal == NULL; + bool32 retiring_head_retained = + log_head_is_equal(spl.checkpoint.sealed_head, retiring_log); + platform_mutex_unlock(&spl.checkpoint_state_lock); + + log_head live_log = log_get_head(spl.log); + bool32 replacement_distinct = !log_head_is_equal(retiring_log, live_log); + bool32 replacement_initially_empty = log_is_empty(spl.log); + superblock_tree_record failed_cut_record; + superblock_get_tree_record(&spl.superblock, &failed_cut_record); + bool32 retiring_log_still_named = + checkpoint_record_names_log(failed_cut_record.live_log, retiring_log); + bool32 no_sealed_log_published = + SUPERBLOCK_NO_LOG(failed_cut_record.sealed_log); + + core_log_append_fault append_fault = {0}; + core_log_append_fault_install(&append_fault, spl.log); + test_key(&keybuf, TEST_RANDOM, 2, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 2, &msg); + message expected = merge_accumulator_to_message(&msg); + platform_status append_rc = + core_insert(&spl, key_buffer_key(&keybuf), expected, NULL); + uint64 append_fault_calls = + __atomic_load_n(&append_fault.calls, __ATOMIC_RELAXED); + core_log_append_fault_uninstall(&append_fault); + + bool32 replacement_still_empty = log_is_empty(spl.log); + lookup_result found; + lookup_result_init( + &found, spl.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + platform_status lookup_rc = + core_lookup(&spl, key_buffer_key(&keybuf), &found); + bool32 failed_update_visible = + SUCCESS(lookup_rc) && lookup_result_found(&found) + && message_lex_cmp( + expected, + merge_accumulator_to_message(lookup_result_accumulator(&found))) + == 0; + lookup_result_deinit(&found); + + platform_status durability_rc = core_durable_barrier(&spl); + + /* Reset the fault budget for both cut retry and root publication. */ + checkpoint_barrier_fault_arm(&data->checkpoint_fault, 0); + platform_status unmount_rc = core_unmount(&spl, FALSE); + uint64 unmount_barriers = + __atomic_load_n(&data->checkpoint_fault.barriers, __ATOMIC_RELAXED); + bool32 handle_remained_mounted = spl.log != NULL; + checkpoint_barrier_fault_uninstall(&data->checkpoint_fault); + + platform_status cleanup_mount_rc = STATUS_OK; + bool32 handle_remained_usable = FALSE; + if (!SUCCESS(unmount_rc)) { + /* A refused non-forced unmount leaves the original handle usable. */ + lookup_result after_unmount; + lookup_result_init( + &after_unmount, spl.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + platform_status after_unmount_rc = + core_lookup(&spl, key_buffer_key(&keybuf), &after_unmount); + handle_remained_usable = + SUCCESS(after_unmount_rc) && lookup_result_found(&after_unmount); + lookup_result_deinit(&after_unmount); + core_destroy(&spl); + } else { + /* Keep a regressed implementation from leaking into fixture teardown. */ + core_handle cleanup; + cleanup_mount_rc = core_mount(&cleanup, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + root_id, + data->hid); + if (SUCCESS(cleanup_mount_rc)) { + core_destroy(&cleanup); + } + } + merge_accumulator_deinit(&msg); + + ASSERT_TRUE(STATUS_IS_EQ(checkpoint_rc, STATUS_IO_ERROR), + "checkpoint returned %s instead of the injected IO error\n", + platform_status_to_string(checkpoint_rc)); + ASSERT_TRUE(SUCCESS(checkpoint_quiesce_rc)); + ASSERT_TRUE(unpublished_sealed_cut, + "failed checkpoint did not leave a sealed unpublished cut\n"); + ASSERT_TRUE(retiring_head_retained, + "failed checkpoint did not retain the sealed log head\n"); + ASSERT_TRUE(replacement_distinct, + "checkpoint did not install a replacement live log\n"); + ASSERT_TRUE(replacement_initially_empty, + "replacement log was not initially empty\n"); + ASSERT_TRUE(retiring_log_still_named, + "failed cut publication did not retain the retiring log\n"); + ASSERT_TRUE(no_sealed_log_published, + "failed cut publication unexpectedly changed the sealed slot\n"); + ASSERT_EQUAL(1, append_fault_calls); + ASSERT_TRUE(STATUS_IS_EQ(append_rc, STATUS_INVALID_STATE), + "injected append returned %s\n", + platform_status_to_string(append_rc)); + ASSERT_TRUE(failed_update_visible, + "failed log append did not leave its memtable update visible\n"); + ASSERT_TRUE(replacement_still_empty, + "failed first append incorrectly marked the log non-empty\n"); + ASSERT_FALSE(SUCCESS(durability_rc), + "durability barrier crossed the poisoned replacement log\n"); + ASSERT_FALSE(SUCCESS(unmount_rc), + "unmount accepted the retiring log despite a poisoned " + "replacement\n"); + ASSERT_TRUE(unmount_barriers >= 2, + "unmount reached only %lu faulted barriers\n", + unmount_barriers); + ASSERT_TRUE(handle_remained_mounted, + "failed non-forced unmount consumed the core handle\n"); + ASSERT_TRUE(handle_remained_usable, + "failed non-forced unmount left the core handle unusable\n"); + ASSERT_TRUE(SUCCESS(cleanup_mount_rc), + "cleanup mount failed after unexpected unmount success: %s\n", + platform_status_to_string(cleanup_mount_rc)); +} + +typedef struct checkpoint_advance_fault_result { + platform_status mkfs_rc; + platform_status initial_quiesce_rc; + platform_status failure_rc; + platform_status failure_quiesce_rc; + platform_status fill_rc; + platform_status advance_rc; + platform_status retry_rc; + platform_status retry_quiesce_rc; + + log_head retired_log; + uint64 barriers_after_failure; + uint64 retired_ref_after_failure; + uint64 retired_ref_after_retry; + bool32 threshold_reached; + + superblock_tree_record failure_record; + superblock_tree_record pre_advance_record; + superblock_tree_record advance_record; + superblock_tree_record retry_record; +} checkpoint_advance_fault_result; + +/* + * Run one bounded, persistent-looking barrier-failure/retry cycle without + * making assertions while the io ops table is overridden. This guarantees + * that the override is kept alive until all checkpoint tasks are quiescent and + * restored even when an observed result is not the expected one. + */ +static checkpoint_advance_fault_result +checkpoint_test_advance_retry(struct CTEST_IMPL_DATA_SNAME(splinter) * data, + uint64 skip_barriers, + bool32 exercise_chained_advance) +{ + checkpoint_advance_fault_result result; + ZERO_STRUCT(result); + + allocator *alp = (allocator *)&data->al; + core_handle spl; + + result.mkfs_rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + if (!SUCCESS(result.mkfs_rc)) { + return result; + } + + result.initial_quiesce_rc = task_perform_until_quiescent(spl.ts); + result.retired_log = log_get_head(spl.log); + + checkpoint_barrier_fault_install(&data->checkpoint_fault, data->io); + checkpoint_barrier_fault_arm(&data->checkpoint_fault, skip_barriers); + + result.failure_rc = core_checkpoint(&spl, 0); + /* + * Leave the fault run armed while draining: a background + * incorporation may be the caller which first reaches COMPLETING. + */ + result.failure_quiesce_rc = task_perform_until_quiescent(spl.ts); + superblock_get_tree_record(&spl.superblock, &result.failure_record); + result.barriers_after_failure = + __atomic_load_n(&data->checkpoint_fault.barriers, __ATOMIC_RELAXED); + result.retired_ref_after_failure = + allocator_get_refcount(alp, result.retired_log.meta_addr); + + if (exercise_chained_advance) { + /* Give the threshold-crossing retry a fresh bounded failure budget. */ + checkpoint_barrier_fault_arm(&data->checkpoint_fault, 0); + /* + * The failed cut's generation was incorporated by the quiesce above. + * While publication is still faulted, fill the current log just far + * enough to cross its extent-based size threshold. The crossing attempt + * fails and leaves the threshold hint set. After disabling the fault, + * exactly one more insert supplies the retry opportunity. Its + * core_checkpoint_advance() call for this failed cut must both republish + * it and notice that completion is already eligible; no later + * incorporation edge is guaranteed to arrive. + */ + platform_assert(spl.cfg.checkpoint_log_size_bytes != 0); + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + for (uint64 i = 0; + i < 30000 + && log_get_size(spl.log) < spl.cfg.checkpoint_log_size_bytes; + i++) + { + test_key( + &keybuf, TEST_RANDOM, 0, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, i, &msg); + result.fill_rc = core_insert(&spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + if (!SUCCESS(result.fill_rc)) { + break; + } + } + result.threshold_reached = + log_get_size(spl.log) >= spl.cfg.checkpoint_log_size_bytes; + superblock_get_tree_record(&spl.superblock, &result.pre_advance_record); + + checkpoint_barrier_fault_disable(&data->checkpoint_fault); + test_key(&keybuf, TEST_RANDOM, 0, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 30000, &msg); + result.advance_rc = core_insert(&spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + merge_accumulator_deinit(&msg); + superblock_get_tree_record(&spl.superblock, &result.advance_record); + } else { + checkpoint_barrier_fault_disable(&data->checkpoint_fault); + } - // Second checkpoint with no new inserts is a clean rotate. - rc = core_checkpoint(&spl, 0); - ASSERT_TRUE(SUCCESS(rc)); + result.retry_rc = core_checkpoint(&spl, 0); + result.retry_quiesce_rc = task_perform_until_quiescent(spl.ts); + superblock_get_tree_record(&spl.superblock, &result.retry_record); + result.retired_ref_after_retry = + allocator_get_refcount(alp, result.retired_log.meta_addr); + /* Restore the real ops before any CTest assertion can leave this scope. */ + checkpoint_barrier_fault_uninstall(&data->checkpoint_fault); core_destroy(&spl); + return result; +} + +/* + * Let the retiring log's durability barrier succeed, then start a long run of + * failures for barriers used to publish the log cut. The durable record must + * continue to name the retiring log as live. Once that generation has already + * been incorporated, one later core_checkpoint_advance() call must publish the + * cut and immediately complete it rather than waiting for a vanished + * incorporation edge. + */ +CTEST2(splinter, test_checkpoint_advance_retries_cut_publish_failure) +{ + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 1; + + checkpoint_advance_fault_result result = + checkpoint_test_advance_retry(data, 1, TRUE); + + ASSERT_TRUE(SUCCESS(result.mkfs_rc)); + ASSERT_TRUE(SUCCESS(result.initial_quiesce_rc)); + ASSERT_TRUE(STATUS_IS_EQ(result.failure_rc, STATUS_IO_ERROR), + "checkpoint returned %s instead of the injected IO error\n", + platform_status_to_string(result.failure_rc)); + ASSERT_TRUE(SUCCESS(result.failure_quiesce_rc)); + ASSERT_TRUE(result.barriers_after_failure >= 2); + ASSERT_TRUE(checkpoint_record_names_log(result.failure_record.live_log, + result.retired_log)); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(result.failure_record.sealed_log)); + ASSERT_NOT_EQUAL(0, result.retired_ref_after_failure); + + ASSERT_TRUE(SUCCESS(result.fill_rc)); + ASSERT_TRUE(result.threshold_reached); + ASSERT_TRUE(checkpoint_record_names_log(result.pre_advance_record.live_log, + result.retired_log)); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(result.pre_advance_record.sealed_log)); + ASSERT_TRUE(SUCCESS(result.advance_rc)); + ASSERT_FALSE(SUPERBLOCK_NO_LOG(result.advance_record.live_log)); + ASSERT_FALSE(checkpoint_record_names_log(result.advance_record.live_log, + result.retired_log)); + ASSERT_FALSE(checkpoint_record_names_log(result.advance_record.sealed_log, + result.retired_log)); + + ASSERT_TRUE(SUCCESS(result.retry_rc), + "checkpoint retry failed: %s\n", + platform_status_to_string(result.retry_rc)); + ASSERT_TRUE(SUCCESS(result.retry_quiesce_rc)); + ASSERT_FALSE(SUPERBLOCK_NO_LOG(result.retry_record.live_log)); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(result.retry_record.sealed_log)); + ASSERT_FALSE(checkpoint_record_names_log(result.retry_record.live_log, + result.retired_log)); +} + +/* + * Once the cut is durable and its generation is incorporated, start a long run + * of root durability-barrier failures in COMPLETING. The sealed log must + * remain reachable and referenced until a later advance call commits the root + * successfully. + */ +CTEST2(splinter, test_checkpoint_advance_retries_completion_failure) +{ + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 0; + + checkpoint_advance_fault_result result = + checkpoint_test_advance_retry(data, 2, FALSE); + + ASSERT_TRUE(SUCCESS(result.mkfs_rc)); + ASSERT_TRUE(SUCCESS(result.initial_quiesce_rc)); + ASSERT_TRUE(STATUS_IS_EQ(result.failure_rc, STATUS_IO_ERROR), + "checkpoint returned %s instead of the injected IO error\n", + platform_status_to_string(result.failure_rc)); + ASSERT_TRUE(SUCCESS(result.failure_quiesce_rc)); + ASSERT_TRUE(result.barriers_after_failure >= 3); + ASSERT_TRUE(checkpoint_record_names_log(result.failure_record.sealed_log, + result.retired_log)); + ASSERT_FALSE(checkpoint_record_names_log(result.failure_record.live_log, + result.retired_log)); + ASSERT_NOT_EQUAL(0, result.retired_ref_after_failure); + + ASSERT_TRUE(SUCCESS(result.retry_rc), + "checkpoint retry failed: %s\n", + platform_status_to_string(result.retry_rc)); + ASSERT_TRUE(SUCCESS(result.retry_quiesce_rc)); + ASSERT_FALSE(SUPERBLOCK_NO_LOG(result.retry_record.live_log)); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(result.retry_record.sealed_log)); + ASSERT_FALSE(checkpoint_record_names_log(result.retry_record.live_log, + result.retired_log)); + ASSERT_EQUAL(0, result.retired_ref_after_retry); } /* @@ -348,7 +2859,7 @@ CTEST2(splinter, test_self_managed_checkpoints_reclaim_log_space) for (uint64 i = 0; i < num_checkpoints; i++) { superblock_get_tree_record(&spl.superblock, &rec); - uint64 retired_meta_addr = rec.live_log.meta_addr; + uint64 retired_meta_addr = rec.live_log.head.meta_addr; ASSERT_NOT_EQUAL(0, retired_meta_addr); rc = core_checkpoint(&spl, 0); @@ -357,7 +2868,7 @@ CTEST2(splinter, test_self_managed_checkpoints_reclaim_log_space) // (a) A cut happened: a different log is now live, and it covers only // generations from the cut onward. superblock_get_tree_record(&spl.superblock, &rec); - ASSERT_NOT_EQUAL(retired_meta_addr, rec.live_log.meta_addr); + ASSERT_NOT_EQUAL(retired_meta_addr, rec.live_log.head.meta_addr); ASSERT_TRUE(SUPERBLOCK_NO_LOG(rec.sealed_log)); // (b) The retired log's space came back. @@ -398,7 +2909,7 @@ run_auto_checkpoint_workload(void *datap, uint64 log_size_threshold) allocator *alp = (allocator *)&data->al; data->system_cfg->splinter_cfg.use_log = TRUE; - // Rotate the log / advance the durable root once the log reaches this size. + // Arm an automatic checkpoint once the log reaches this size. data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = log_size_threshold; @@ -413,6 +2924,9 @@ run_auto_checkpoint_workload(void *datap, uint64 log_size_threshold) data->hid); ASSERT_TRUE(SUCCESS(rc)); + superblock_tree_record initial_rec; + superblock_get_tree_record(&spl.superblock, &initial_rec); + uint64 num_inserts = splinter_do_inserts(data, &spl, FALSE, NULL); ASSERT_NOT_EQUAL(0, num_inserts); @@ -421,16 +2935,13 @@ run_auto_checkpoint_workload(void *datap, uint64 log_size_threshold) ASSERT_TRUE(SUCCESS(rc)); if (log_size_threshold != 0) { - // The inserts must have driven at least one automatic checkpoint through - // to completion. - ASSERT_NOT_EQUAL(0, spl.checkpoint.completions); - - // At rest a checkpoint has either completed (IDLE) or been armed for a - // rotation that idle never triggered (PENDING); both leave no sealed log. - ASSERT_TRUE(spl.checkpoint.phase == CORE_CHECKPOINT_IDLE - || spl.checkpoint.phase == CORE_CHECKPOINT_PENDING); superblock_tree_record rec; superblock_get_tree_record(&spl.superblock, &rec); + + // The inserts drove at least one complete automatic log rotation. + ASSERT_FALSE(SUPERBLOCK_NO_LOG(rec.live_log)); + ASSERT_FALSE( + checkpoint_records_name_same_log(initial_rec.live_log, rec.live_log)); ASSERT_TRUE(SUPERBLOCK_NO_LOG(rec.sealed_log)); // A checkpoint published an advanced, incorporated durable root mid-run: // at least one generation was folded in, so the first unincorporated @@ -472,6 +2983,213 @@ CTEST2(splinter, test_auto_checkpoint) run_auto_checkpoint_workload(data, 2 * data->system_cfg->io_cfg.extent_size); } +/* + * Crossing the soft log-size threshold should arm a checkpoint without + * immediately forcing a memtable rotation. If the memtable subsequently + * fills during the byte grace period, the ordinary insert-time rotation must + * consume that pending checkpoint and cut the log exactly once. + * + * Repeatedly overwriting one key gets the log to the soft threshold without + * filling the memtable. Once PENDING is observed, lower the live memtable's + * extent ceiling to one beyond its current allocation. Distinct inserts then + * take the normal fullness-triggered path in + * memtable_maybe_rotate_and_begin_insert; no timing or scheduler assumptions + * are involved. + */ +CTEST2(splinter, test_auto_checkpoint_grace_allows_natural_rotation) +{ + allocator *alp = (allocator *)&data->al; + uint64 extent_size = data->system_cfg->io_cfg.extent_size; + uint64 grace_bytes = 64 * extent_size; + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = extent_size; + data->system_cfg->splinter_cfg.checkpoint_log_grace_bytes = grace_bytes; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 start_generation = memtable_generation(&spl.mt_ctxt); + log_head start_log = log_get_head(spl.log); + + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + + bool32 pending = FALSE; + for (uint64 i = 0; i < 30000 && !pending; i++) { + test_key(&keybuf, TEST_RANDOM, 0, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, i, &msg); + rc = core_insert(&spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + ASSERT_TRUE(SUCCESS(rc)); + + platform_mutex_lock(&spl.checkpoint_state_lock); + pending = spl.checkpoint.phase == CORE_CHECKPOINT_PENDING; + platform_mutex_unlock(&spl.checkpoint_state_lock); + } + ASSERT_TRUE(pending, "soft threshold did not arm a checkpoint\n"); + + uint64 arm_size; + uint64 force_at_log_size; + platform_mutex_lock(&spl.checkpoint_state_lock); + arm_size = log_get_size(spl.log); + force_at_log_size = spl.checkpoint.force_at_log_size; + platform_mutex_unlock(&spl.checkpoint_state_lock); + + ASSERT_EQUAL(start_generation, memtable_generation(&spl.mt_ctxt)); + ASSERT_TRUE(log_head_is_equal(start_log, log_get_head(spl.log))); + ASSERT_EQUAL(arm_size + grace_bytes, force_at_log_size); + ASSERT_TRUE(arm_size < force_at_log_size); + + /* + * Make this memtable fill after allocating one more extent. The fresh + * successor starts below this limit, avoiding an artificial rotation loop. + */ + uint64 active_index = start_generation % spl.mt_ctxt.cfg.max_memtables; + uint64 original_max_extents = spl.mt_ctxt.cfg.max_extents_per_memtable; + spl.mt_ctxt.cfg.max_extents_per_memtable = + mini_num_extents(&spl.mt_ctxt.mt[active_index].mini) + 1; + + bool32 rotated = FALSE; + for (uint64 i = 1; i < 30000 && !rotated; i++) { + ASSERT_TRUE(log_get_size(spl.log) < force_at_log_size, + "log exhausted grace before natural rotation\n"); + test_key(&keybuf, TEST_RANDOM, i, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 30000 + i, &msg); + rc = core_insert(&spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + ASSERT_TRUE(SUCCESS(rc)); + rotated = memtable_generation(&spl.mt_ctxt) != start_generation; + } + spl.mt_ctxt.cfg.max_extents_per_memtable = original_max_extents; + + ASSERT_TRUE(rotated, "memtable never rotated naturally\n"); + ASSERT_EQUAL(start_generation + 1, memtable_generation(&spl.mt_ctxt)); + ASSERT_FALSE(log_head_is_equal(start_log, log_get_head(spl.log))); + platform_mutex_lock(&spl.checkpoint_state_lock); + pending = spl.checkpoint.phase == CORE_CHECKPOINT_PENDING; + platform_mutex_unlock(&spl.checkpoint_state_lock); + ASSERT_FALSE(pending, + "natural rotation did not consume the pending checkpoint\n"); + + rc = task_perform_until_quiescent(spl.ts); + ASSERT_TRUE(SUCCESS(rc)); + merge_accumulator_deinit(&msg); + core_destroy(&spl); +} + +/* + * An overwrite-only workload does not fill its memtable, so after the grace + * bytes are consumed the automatic policy must force the pending rotation. + * The test observes the per-PENDING byte deadline directly and verifies that + * every insert below it leaves both the generation and live log unchanged. + */ +CTEST2(splinter, test_auto_checkpoint_grace_forces_overwrite_rotation) +{ + allocator *alp = (allocator *)&data->al; + uint64 extent_size = data->system_cfg->io_cfg.extent_size; + uint64 grace_bytes = 2 * extent_size; + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = extent_size; + data->system_cfg->splinter_cfg.checkpoint_log_grace_bytes = grace_bytes; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 start_generation = memtable_generation(&spl.mt_ctxt); + log_head start_log = log_get_head(spl.log); + + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + + bool32 pending = FALSE; + uint64 value_i = 0; + for (; value_i < 30000 && !pending; value_i++) { + test_key(&keybuf, TEST_RANDOM, 0, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, value_i, &msg); + rc = core_insert(&spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + ASSERT_TRUE(SUCCESS(rc)); + + platform_mutex_lock(&spl.checkpoint_state_lock); + pending = spl.checkpoint.phase == CORE_CHECKPOINT_PENDING; + platform_mutex_unlock(&spl.checkpoint_state_lock); + } + ASSERT_TRUE(pending, "soft threshold did not arm a checkpoint\n"); + + uint64 arm_size; + uint64 force_at_log_size; + platform_mutex_lock(&spl.checkpoint_state_lock); + arm_size = log_get_size(spl.log); + force_at_log_size = spl.checkpoint.force_at_log_size; + platform_mutex_unlock(&spl.checkpoint_state_lock); + + ASSERT_EQUAL(start_generation, memtable_generation(&spl.mt_ctxt)); + ASSERT_TRUE(log_head_is_equal(start_log, log_get_head(spl.log))); + ASSERT_EQUAL(arm_size + grace_bytes, force_at_log_size); + ASSERT_TRUE(arm_size < force_at_log_size); + + bool32 rotated = FALSE; + for (uint64 i = 0; i < 30000 && !rotated; i++, value_i++) { + uint64 size_before = log_get_size(spl.log); + ASSERT_TRUE(size_before < force_at_log_size, + "pending checkpoint remained unrotated at hard limit: " + "size=%lu force_at=%lu\n", + size_before, + force_at_log_size); + + test_key(&keybuf, TEST_RANDOM, 0, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, value_i, &msg); + rc = core_insert(&spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + ASSERT_TRUE(SUCCESS(rc)); + + rotated = memtable_generation(&spl.mt_ctxt) != start_generation; + if (!rotated) { + ASSERT_TRUE(log_head_is_equal(start_log, log_get_head(spl.log))); + ASSERT_TRUE(log_get_size(spl.log) < force_at_log_size, + "policy did not force at byte deadline: size=%lu " + "force_at=%lu\n", + log_get_size(spl.log), + force_at_log_size); + } + } + + ASSERT_TRUE(rotated, "overwrite workload never forced a rotation\n"); + ASSERT_EQUAL(start_generation + 1, memtable_generation(&spl.mt_ctxt)); + ASSERT_FALSE(log_head_is_equal(start_log, log_get_head(spl.log))); + + rc = task_perform_until_quiescent(spl.ts); + ASSERT_TRUE(SUCCESS(rc)); + merge_accumulator_deinit(&msg); + core_destroy(&spl); +} + /* * The reason the policy is sized in log bytes rather than memtable generations. * @@ -487,7 +3205,9 @@ CTEST2(splinter, test_auto_checkpoint_on_overwrites) data->system_cfg->splinter_cfg.use_log = TRUE; data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 2 * data->system_cfg->io_cfg.extent_size; - // Also verify the reported checkpoint count against the internal one. + data->system_cfg->splinter_cfg.checkpoint_log_grace_bytes = + 2 * data->system_cfg->io_cfg.extent_size; + // Also verify that the public checkpoint statistic is updated. data->system_cfg->splinter_cfg.use_stats = TRUE; core_handle spl; @@ -501,6 +3221,9 @@ CTEST2(splinter, test_auto_checkpoint_on_overwrites) data->hid); ASSERT_TRUE(SUCCESS(rc)); + superblock_tree_record initial_rec; + superblock_get_tree_record(&spl.superblock, &initial_rec); + uint64 start_generation = memtable_generation(&spl.mt_ctxt); // Hammer a single key. Enough writes to push the log well past one extent. @@ -526,28 +3249,28 @@ CTEST2(splinter, test_auto_checkpoint_on_overwrites) * policy would have had nothing to trigger on: any generation advance here * came from a checkpoint forcing a rotation, not from the memtable filling. */ - ASSERT_NOT_EQUAL(0, - spl.checkpoint.completions, - "overwrite-only workload did not trigger a checkpoint; " - "generation went %lu -> %lu\n", - start_generation, - memtable_generation(&spl.mt_ctxt)); - - /* - * The reported statistic must agree with the machinery's own count: it is - * summed across threads, so this catches both a missed increment and a - * double count. - */ + superblock_tree_record rec; + superblock_get_tree_record(&spl.superblock, &rec); + ASSERT_FALSE(SUPERBLOCK_NO_LOG(rec.live_log)); + ASSERT_FALSE( + checkpoint_records_name_same_log(initial_rec.live_log, rec.live_log), + "overwrite-only workload did not rotate the log; generation went %lu " + "-> %lu\n", + start_generation, + memtable_generation(&spl.mt_ctxt)); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(rec.sealed_log)); + + /* The statistic is per-thread, so sum it across every registered thread. */ uint64 reported = 0; for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { reported += spl.stats[thr_i].checkpoints_completed; } - ASSERT_EQUAL(spl.checkpoint.completions, - reported, - "checkpoints_completed stat (%lu) disagrees with the " - "checkpoint state's count (%lu)\n", - reported, - spl.checkpoint.completions); + ASSERT_NOT_EQUAL(0, + reported, + "overwrite-only workload did not complete a checkpoint; " + "generation went %lu -> %lu\n", + start_generation, + memtable_generation(&spl.mt_ctxt)); // The surviving value must be the last one written. lookup_result qdata; @@ -568,17 +3291,186 @@ CTEST2(splinter, test_auto_checkpoint_on_overwrites) } /* - * The second checkpoint slot is a torn-write fallback, not permission for a - * normal mount to silently roll back past a newer, valid active record. A - * successful mount publishes such an active record; until crash recovery is - * implemented, a concurrent/restarted normal mount must reject it even - * though the preceding clean record remains valid in the other slot. + * The crash-recovery refcount rebuild has to reconstruct, from the tree alone, + * exactly the map that normal operation maintained -- so this checks it against + * the one authority on the subject: the map a clean unmount persisted. + * + * Comparing whole maps rather than spot-checking a few extents is the point. An + * extent the walk misses leaks, and one it counts twice is freed while still in + * use; both show up here as a mismatched refcount, and nothing else in the + * suite would notice either. + * + * Coverage note: the default configuration builds a tree of one node, which + * exercises the per-branch and per-filter accounting but never the descent. The + * height a tree reaches is driven by how much data it holds and not by the + * memtable size, so reaching a second level needs a large run -- at + * --num-inserts 20000000 this walks 36 nodes, and so also covers descending, + * the branches shared between nodes, and the guard against descending twice. + * That is too slow to make the default, hence this note. + */ +CTEST2(splinter, test_recover_allocations_reproduces_persisted_map) +{ + allocator *alp = (allocator *)&data->al; + allocator_config *acfg = allocator_get_config(alp); + allocator_root_id root_id = test_generate_allocator_root_id(); + core_handle spl; + platform_status rc; + + rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + /* + * A real tree, not the empty root: the walk is only interesting once there + * are interior nodes, several bundles per node, and branches that more than + * one node references. + */ + splinter_do_inserts(data, &spl, FALSE, NULL); + + rc = core_unmount(&spl, FALSE); + ASSERT_TRUE(SUCCESS(rc)); + + /* Read the durable record the way a mount would. */ + superblock_context sb; + rc = superblock_context_init(&sb, data->io, acfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_TRUE(SUCCESS(superblock_mount(&sb, acfg))); + superblock_tree_record rec; + superblock_get_tree_record(&sb, &rec); + // A clean unmount: the persisted map is trustworthy and the logs are gone. + ASSERT_TRUE(superblock_allocation_state_valid(&sb)); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(rec.live_log)); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(rec.sealed_log)); + ASSERT_NOT_EQUAL(0, rec.root_addr); + superblock_context_deinit(&sb); + + /* + * Ground truth. core_unmount() persisted this very map, so the in-memory + * copy still standing here is byte-for-byte what a clean mount would load. + */ + uint64 extent_size = acfg->io_cfg->extent_size; + uint64 num_extents = allocator_get_capacity(alp) / extent_size; + refcount *expected = TYPED_ARRAY_MALLOC(data->hid, expected, num_extents); + ASSERT_NOT_NULL(expected); + uint64 num_referenced = 0; + for (uint64 i = 0; i < num_extents; i++) { + expected[i] = allocator_get_refcount(alp, i * extent_size); + if (expected[i] != AL_FREE) { + num_referenced++; + } + } + + /* + * Rebuild it -- twice. + * + * Once for the obvious reason, and a second time because recovery itself + * rebuilds twice: once counting the logs so replay is not handed their + * space, and again from the root alone afterwards, which is what releases + * it. A second rebuild that drifted from the first would mean recovery + * silently leaking or double-freeing on every crash, so the round it runs in + * has to make no difference at all. + * + * The first rebuild starts from a freshly attached allocator, matching a + * real mount; the second runs against the map the first one left behind, + * which is the case recovery actually depends on. The cache keeps pointing + * at the same allocator struct throughout, which is how the branch and + * filter walks reach it. + */ + rc_allocator_deinit(&data->al); + rc = rc_allocator_mount( + &data->al, acfg, data->io, data->hid, platform_get_module_id()); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 mismatches = 0; + for (uint64 round = 0; round < 2; round++) { + ASSERT_TRUE(SUCCESS(allocator_recovery_begin(alp))); + rc = trunk_recover_allocations( + data->system_cfg->splinter_cfg.trunk_node_cfg, + (cache *)data->clock_cache, + data->hid, + rec.root_addr); + ASSERT_TRUE(SUCCESS(rc)); + allocator_recovery_finish(alp); + + for (uint64 i = 0; i < num_extents; i++) { + refcount actual = allocator_get_refcount(alp, i * extent_size); + if (actual != expected[i]) { + if (mismatches < 16) { + platform_error_log("round %lu: extent %lu (addr %lu): persisted " + "refcount %u, rebuilt %u\n", + round, + i, + i * extent_size, + expected[i], + actual); + } + mismatches++; + } + } + // The count the allocator reports has to be rebuilt too, not accumulated. + ASSERT_EQUAL(num_referenced, + allocator_in_use(alp), + "round %lu: the allocator reports %lu extents in use, but " + "%lu are referenced\n", + round, + allocator_in_use(alp), + num_referenced); + } + platform_free(data->hid, expected); + + ASSERT_EQUAL(0, + mismatches, + "the rebuilt map differs from the persisted one in %lu of %lu " + "extents\n", + mismatches, + num_extents); + // Guard against the comparison passing because there was nothing to compare. + ASSERT_TRUE(num_referenced > 1, + "only %lu extents were referenced; the tree is too small for " + "this test to mean anything\n", + num_referenced); + + /* + * Leave nothing behind for the fixture's leak check: the map still holds + * every reference the tree needs, so erase the tree. Mounting reloads the + * persisted map over the rebuilt one, which the comparison above has just + * shown to be the same map. + */ + core_handle cleanup; + rc = core_mount(&cleanup, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + core_destroy(&cleanup); +} + +/* + * A conservative reference-release failure can leave the live allocator map + * usable but inexact. Unmount must still succeed once the root is durable, + * while refusing to bless that map as clean allocation state. The next mount + * then rebuilds it, after which an ordinary clean unmount may persist it again. + * + * trunk_snapshot_release() has no deterministic failure injection today, so + * set the core's sticky result bit directly to exercise the shutdown/recovery + * contract that such a failure triggers. */ -CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) +CTEST2(splinter, test_unmount_skips_allocator_map_that_needs_rebuild) { allocator *alp = (allocator *)&data->al; + allocator_config *acfg = allocator_get_config(alp); allocator_root_id root_id = test_generate_allocator_root_id(); - core_handle created, mounted, rejected, cleanup; + core_handle created, recovered, cleanup; platform_status rc; rc = core_mkfs(&created, @@ -591,7 +3483,6 @@ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) data->hid); ASSERT_TRUE(SUCCESS(rc)); - /* Give the clean record a real COW root, not just the empty-tree root. */ DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); merge_accumulator msg; merge_accumulator_init(&msg, data->hid); @@ -604,11 +3495,51 @@ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) merge_accumulator_deinit(&msg); ASSERT_TRUE(SUCCESS(rc)); - rc = core_unmount(&created); + created.allocator_map_needs_rebuild = TRUE; + rc = core_unmount(&created, FALSE); ASSERT_TRUE(SUCCESS(rc)); - /* This mount advances the A/B sequence with an unmounted=FALSE record. */ - rc = core_mount(&mounted, + superblock_context sb; + rc = superblock_context_init(&sb, data->io, acfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_TRUE(SUCCESS(superblock_mount(&sb, acfg))); + ASSERT_FALSE(superblock_allocation_state_valid(&sb)); + superblock_context_deinit(&sb); + + rc = core_mount(&recovered, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_FALSE(recovered.allocator_map_needs_rebuild); + + lookup_result qdata; + lookup_result_init( + &qdata, recovered.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + rc = core_lookup(&recovered, key_buffer_key(&keybuf), &qdata); + ASSERT_TRUE(SUCCESS(rc)); + verify_tuple(&recovered, + &data->gen, + 1, + key_buffer_key(&keybuf), + merge_accumulator_to_message(lookup_result_accumulator(&qdata)), + TRUE); + lookup_result_deinit(&qdata); + + rc = core_unmount(&recovered, FALSE); + ASSERT_TRUE(SUCCESS(rc)); + + rc = superblock_context_init(&sb, data->io, acfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_TRUE(SUCCESS(superblock_mount(&sb, acfg))); + ASSERT_TRUE(superblock_allocation_state_valid(&sb)); + superblock_context_deinit(&sb); + + rc = core_mount(&cleanup, &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, @@ -617,8 +3548,67 @@ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) root_id, data->hid); ASSERT_TRUE(SUCCESS(rc)); + core_destroy(&cleanup); +} + +/* + * The second checkpoint slot is a torn-write fallback, not permission for a + * normal mount to silently roll back past a newer, valid active record. A + * successful mount publishes such an active record, so this walks a record pair + * through the states that produces -- clean, then active, then clean again -- + * and requires it to stay mountable throughout. + * + * It used to also require that a second mount over the active record be + * *rejected*, which was true only while crash recovery was unimplemented: an + * invalid allocation state now sends a mount into recovery rather than into an + * error, which is the whole point of it. Two consequences worth knowing: + * + * - Recovering the newer active record, rather than refusing it, is what + * actually honours the no-rollback rule. Exercising that needs a genuine + * crash, because recovery rebuilds the refcount map from scratch and so + * invalidates the accounting of any handle still holding the instance -- + * which no test can arrange from inside this fixture. It belongs with the + * process-level crash tests. + * - Nothing now stops a second mount of a *live* instance. The clean-only + * rule used to prevent that as a side effect; distinguishing "crashed" from + * "mounted by someone else" needs a marker of its own, since the allocation + * state means only the former. + */ +CTEST2(splinter, test_mount_active_checkpoint_stays_mountable) +{ + allocator *alp = (allocator *)&data->al; + allocator_root_id root_id = test_generate_allocator_root_id(); + core_handle created, mounted, cleanup; + platform_status rc; + + rc = core_mkfs(&created, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + /* Give the clean record a real COW root, not just the empty-tree root. */ + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + test_key(&keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 1, &msg); + rc = core_insert(&created, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + merge_accumulator_deinit(&msg); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_unmount(&created, FALSE); + ASSERT_TRUE(SUCCESS(rc)); - rc = core_mount(&rejected, + /* This mount advances the A/B sequence with an unmounted=FALSE record. */ + rc = core_mount(&mounted, &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, @@ -626,10 +3616,10 @@ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) &data->tasks, root_id, data->hid); - ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); + ASSERT_TRUE(SUCCESS(rc)); /* Finish cleanly, then prove the same record pair is mountable again. */ - rc = core_unmount(&mounted); + rc = core_unmount(&mounted, FALSE); ASSERT_TRUE(SUCCESS(rc)); rc = core_mount(&cleanup, diff --git a/tests/unit/splinterdb_forked_child_test.c b/tests/unit/splinterdb_forked_child_test.c index 4a06a04c..5bc2130d 100644 --- a/tests/unit/splinterdb_forked_child_test.c +++ b/tests/unit/splinterdb_forked_child_test.c @@ -144,7 +144,7 @@ CTEST2(splinterdb_forked_child, test_data_structures_handles) // We would get assertions tripping from BTree iterator code here, // if the fix in platform_buffer_create_mmap() to use MAP_SHARED // was not in-place. - splinterdb_close(&spl_handle); + splinterdb_close(&spl_handle, FALSE); } else { // Child should not attempt to run the rest of the tests exit(0); @@ -241,7 +241,7 @@ CTEST2(splinterdb_forked_child, test_one_insert_then_close_bug) // We would get assertions tripping from BTree iterator code here, // if the fix in platform_buffer_create_mmap() to use MAP_SHARED // was not in-place. - splinterdb_close(&spl_handle); + splinterdb_close(&spl_handle, FALSE); } else { platform_deregister_thread(); // child should not attempt to run the rest of the tests @@ -365,7 +365,7 @@ CTEST2(splinterdb_forked_child, " Resuming parent ...\n", platform_get_os_pid(), platform_get_tid()); - splinterdb_close(&spl_handle); + splinterdb_close(&spl_handle, FALSE); } /* @@ -489,7 +489,7 @@ CTEST2(splinterdb_forked_child, test_multiple_forked_process_doing_IOs) platform_get_os_pid(), platform_get_tid()); - splinterdb_close(&spl_handle); + splinterdb_close(&spl_handle, FALSE); } } diff --git a/tests/unit/splinterdb_optimize_test.c b/tests/unit/splinterdb_optimize_test.c index 215e8219..0adb29d6 100644 --- a/tests/unit/splinterdb_optimize_test.c +++ b/tests/unit/splinterdb_optimize_test.c @@ -84,7 +84,7 @@ CTEST_SETUP(splinterdb_optimize) CTEST_TEARDOWN(splinterdb_optimize) { if (data->kvsb != NULL) { - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); } platform_deregister_thread(); } @@ -127,7 +127,7 @@ CTEST2(splinterdb_optimize, test_blocking_with_no_background_threads) { const uint32 num_keys = 320; - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); data->cfg.num_memtable_bg_threads = 0; data->cfg.num_normal_bg_threads = 0; @@ -152,7 +152,7 @@ CTEST2(splinterdb_optimize, test_open_reads_disk_geometry) const uint32 num_keys = 160; load_key_batches(data->kvsb, num_keys, 40); - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); data->cfg.disk_size = 0; data->cfg.page_size = 0; @@ -284,9 +284,14 @@ create_optimize_cfg(splinterdb_config *out_cfg, data_config *default_data_cfg) static void force_flush_current_memtable(splinterdb *kvsb) { - core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); - memtable_force_rotation(&core->mt_ctxt); - platform_status rc = task_perform_until_quiescent(core->ts); + core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); + platform_status rc = task_perform_until_quiescent(core->ts); + ASSERT_TRUE(SUCCESS(rc)); + + rc = memtable_force_rotation(&core->mt_ctxt, NULL); + ASSERT_TRUE(SUCCESS(rc)); + + rc = task_perform_until_quiescent(core->ts); ASSERT_TRUE(SUCCESS(rc)); } diff --git a/tests/unit/splinterdb_quick_test.c b/tests/unit/splinterdb_quick_test.c index 4eb29e23..fbd390cd 100644 --- a/tests/unit/splinterdb_quick_test.c +++ b/tests/unit/splinterdb_quick_test.c @@ -178,7 +178,7 @@ CTEST_SETUP(splinterdb_quick) CTEST_TEARDOWN(splinterdb_quick) { if (data->kvsb) { - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); } platform_deregister_thread(); } @@ -192,6 +192,32 @@ CTEST_TEARDOWN(splinterdb_quick) * The 2nd term is the test-case name, e.g., 'test_basic_flow'. * *********************************************************************** */ +CTEST2(splinterdb_quick, test_checkpoint_log_grace_default) +{ + const core_handle *core = splinterdb_get_trunk_handle(data->kvsb); + ASSERT_EQUAL(2 * core->cfg.trunk_node_cfg->incorporation_size_kv_bytes, + core->cfg.checkpoint_log_grace_bytes); +} + +CTEST2(splinterdb_quick, test_durable_barrier) +{ + // Re-create the instance with logging enabled (SETUP created it without). + int rc = splinterdb_close(&data->kvsb, FALSE); + ASSERT_EQUAL(0, rc); + data->cfg.use_log = TRUE; + rc = splinterdb_create(&data->cfg, &data->kvsb); + ASSERT_EQUAL(0, rc); + + slice key = slice_create(strlen("durable-key"), "durable-key"); + slice value = slice_create(strlen("durable-value"), "durable-value"); + + rc = splinterdb_insert(data->kvsb, key, value, NULL); + ASSERT_EQUAL(0, rc); + + rc = splinterdb_durable_barrier(data->kvsb); + ASSERT_EQUAL(0, rc); +} + /* * * Basic test case that exercises and validates the basic flow of the @@ -373,7 +399,7 @@ CTEST2(splinterdb_quick, test_value_size_gt_max_value_size) splinterdb_lookup_result_deinit(&result); - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); rc = splinterdb_open(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); @@ -1225,7 +1251,8 @@ CTEST2(splinterdb_quick, test_close_and_reopen) ASSERT_EQUAL(0, rc); // Close and re-open the database - splinterdb_close(&data->kvsb); + rc = splinterdb_close(&data->kvsb, FALSE); + ASSERT_EQUAL(0, rc); rc = splinterdb_open(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); @@ -1259,9 +1286,10 @@ CTEST2(splinterdb_quick, test_close_and_reopen) CTEST2(splinterdb_quick, test_logged_close_and_reopen) { // Re-create the instance with logging enabled (SETUP created it without). - splinterdb_close(&data->kvsb); + int rc = splinterdb_close(&data->kvsb, FALSE); + ASSERT_EQUAL(0, rc); data->cfg.use_log = TRUE; - int rc = splinterdb_create(&data->cfg, &data->kvsb); + rc = splinterdb_create(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); slice user_key = slice_create(strlen("logged-key"), "logged-key"); @@ -1271,7 +1299,8 @@ CTEST2(splinterdb_quick, test_logged_close_and_reopen) splinterdb_insert(data->kvsb, user_key, slice_create(val_len, val), NULL); ASSERT_EQUAL(0, rc); - splinterdb_close(&data->kvsb); + rc = splinterdb_close(&data->kvsb, FALSE); + ASSERT_EQUAL(0, rc); rc = splinterdb_open(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); @@ -1313,7 +1342,8 @@ CTEST2(splinterdb_quick, test_repeated_insert_close_reopen) NULL); ASSERT_EQUAL(0, rc, "Insert is expected to pass, iter=%d.", i); - splinterdb_close(&data->kvsb); + rc = splinterdb_close(&data->kvsb, FALSE); + ASSERT_EQUAL(0, rc); rc = splinterdb_open(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); @@ -1326,7 +1356,7 @@ CTEST2(splinterdb_quick, test_custom_data_config) { // We need to reconfigure Splinter with user-specified data_config // Tear down default instance, and create a new one. - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); data->cfg.data_cfg = test_data_config; int rc = splinterdb_create(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); @@ -1426,7 +1456,7 @@ CTEST2(splinterdb_quick, test_existence_only_memtable_lookup) CTEST2(splinterdb_quick, test_existence_only_trunk_lookup_skips_branches) { - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); create_default_cfg(&data->cfg, &data->default_data_cfg.super); data->cfg.use_stats = 1; @@ -1439,7 +1469,7 @@ CTEST2(splinterdb_quick, test_existence_only_trunk_lookup_skips_branches) rc = splinterdb_insert(data->kvsb, user_key, value, NULL); ASSERT_EQUAL(0, rc); - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); rc = splinterdb_open(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); @@ -1538,7 +1568,7 @@ CTEST2(splinterdb_quick, test_write_api_old_result_existence_only) CTEST2(splinterdb_quick, test_write_api_old_result_custom_merge_semantics) { - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); data->cfg.data_cfg = test_data_config; int rc = splinterdb_create(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); @@ -1587,7 +1617,7 @@ CTEST2(splinterdb_quick, test_write_api_old_result_custom_merge_semantics) CTEST2(splinterdb_quick, test_write_api_old_result_merges_memtable_and_trunk) { - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); data->cfg.data_cfg = test_data_config; int rc = splinterdb_create(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); @@ -1599,7 +1629,7 @@ CTEST2(splinterdb_quick, test_write_api_old_result_merges_memtable_and_trunk) rc = splinterdb_insert(data->kvsb, user_key, msg_slice, NULL); ASSERT_EQUAL(0, rc); - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); rc = splinterdb_open(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); @@ -1637,7 +1667,7 @@ CTEST2(splinterdb_quick, test_write_api_old_result_respects_trunk_delete_shadow) int rc = splinterdb_insert(data->kvsb, user_key, value0, NULL); ASSERT_EQUAL(0, rc); - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); rc = splinterdb_open(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); @@ -1659,7 +1689,7 @@ CTEST2(splinterdb_quick, test_iterator_custom_comparator) { // We need to reconfigure Splinter with user-specified key comparator fn. // Tear down default instance, and create a new one. - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); data->default_data_cfg.super.key_compare = custom_key_comparator; data->default_data_cfg.num_comparisons = 0; @@ -1708,7 +1738,7 @@ CTEST2(splinterdb_quick, test_iterator_init_bug) { // We need to reconfigure Splinter with user-specified data_config // Tear down default instance, and create a new one. - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); data->cfg.data_cfg = test_data_config; int rc = splinterdb_create(&data->cfg, &data->kvsb); @@ -1755,7 +1785,7 @@ CTEST2(splinterdb_quick, test_iterator_init_bug) */ CTEST2(splinterdb_quick, test_splinterdb_create_w_background_threads) { - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); default_data_config_init(&data->default_data_cfg.super); create_default_cfg(&data->cfg, &data->default_data_cfg.super); @@ -1776,7 +1806,7 @@ CTEST2(splinterdb_quick, test_splinterdb_create_w_background_threads) */ CTEST2(splinterdb_quick, test_splinterdb_create_w_all_background_threads) { - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); default_data_config_init(&data->default_data_cfg.super); create_default_cfg(&data->cfg, &data->default_data_cfg.super); @@ -1809,9 +1839,16 @@ create_default_cfg(splinterdb_config *out_cfg, data_config *default_data_cfg) static uint64 force_flush_current_memtable(splinterdb *kvsb) { - core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); - uint64 generation = memtable_force_rotation(&core->mt_ctxt); - platform_status rc = task_perform_until_quiescent(core->ts); + core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); + + platform_status rc = task_perform_until_quiescent(core->ts); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 generation; + rc = memtable_force_rotation(&core->mt_ctxt, &generation); + ASSERT_TRUE(SUCCESS(rc)); + + rc = task_perform_until_quiescent(core->ts); ASSERT_TRUE(SUCCESS(rc)); return generation; } diff --git a/tests/unit/splinterdb_stress_test.c b/tests/unit/splinterdb_stress_test.c index c3b5b4be..68e26e52 100644 --- a/tests/unit/splinterdb_stress_test.c +++ b/tests/unit/splinterdb_stress_test.c @@ -66,7 +66,7 @@ CTEST_SETUP(splinterdb_stress) // Optional teardown function for suite, called after every test in suite CTEST_TEARDOWN(splinterdb_stress) { - splinterdb_close(&data->kvsb); + splinterdb_close(&data->kvsb, FALSE); platform_deregister_thread(); } diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index b4f49324..432c0767 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -126,7 +126,7 @@ CTEST2(superblock, test_snapshot_persists_state) ASSERT_TRUE(SUCCESS(rc)); superblock_log_head live = { - .addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}; + .head = {.addr = 0x6000, .meta_addr = 0x8000, .nonce = {.low = 0x11}}}; superblock_log_cut(&ctx, live); superblock_snapshot_tree(&ctx, 0x4000, 0); rc = superblock_make_durable(&ctx); @@ -150,9 +150,9 @@ CTEST2(superblock, test_snapshot_persists_state) superblock_tree_record got; superblock_get_tree_record(&ctx, &got); ASSERT_EQUAL(0x4000, got.root_addr); - ASSERT_EQUAL(0x6000, got.live_log.addr); - ASSERT_EQUAL(0x8000, got.live_log.meta_addr); - ASSERT_EQUAL(0x11, got.live_log.magic); + ASSERT_EQUAL(0x6000, got.live_log.head.addr); + ASSERT_EQUAL(0x8000, got.live_log.head.meta_addr); + ASSERT_EQUAL(0x11, got.live_log.head.nonce.low); ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); // no checkpoint in progress superblock_context_deinit(&ctx); } @@ -161,14 +161,12 @@ CTEST2(superblock, test_snapshot_persists_state) * Steady, begin-checkpoint, and complete-checkpoint tree-record states. L1 * covers generations 0..5 and is cut at 5, so L2 takes over at 6. */ -static const superblock_log_head TEST_LOG_L1 = {.addr = 0x6000, - .meta_addr = 0x8000, - .magic = 0x11, - .start_generation = 0}; -static const superblock_log_head TEST_LOG_L2 = {.addr = 0x10000, - .meta_addr = 0x12000, - .magic = 0x22, - .start_generation = 6}; +static const superblock_log_head TEST_LOG_L1 = { + .head = {.addr = 0x6000, .meta_addr = 0x8000, .nonce = {.low = 0x11}}, + .start_generation = 0}; +static const superblock_log_head TEST_LOG_L2 = { + .head = {.addr = 0x10000, .meta_addr = 0x12000, .nonce = {.low = 0x22}}, + .start_generation = 6}; /* * Walk the two-log checkpoint state machine through the superblock and confirm @@ -198,8 +196,8 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) superblock_tree_record got; superblock_get_tree_record(&ctx, &got); - ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.sealed_log.meta_addr); - ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L1.head.meta_addr, got.sealed_log.head.meta_addr); + ASSERT_EQUAL(TEST_LOG_L2.head.meta_addr, got.live_log.head.meta_addr); // Complete: advance the root past L1's coverage (first unincorporated 9 >= // L2's start 6), so the sealed log is dropped and L2 carries forward. @@ -217,7 +215,7 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) superblock_get_tree_record(&ctx, &got); ASSERT_EQUAL(0x4400, got.root_addr); ASSERT_EQUAL(9, got.first_unincorporated_generation); - ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L2.head.meta_addr, got.live_log.head.meta_addr); ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); superblock_context_deinit(&ctx); } @@ -250,8 +248,8 @@ CTEST2(superblock, test_snapshot_preserves_unincorporated_sealed_log) ASSERT_TRUE(SUCCESS(rc)); superblock_tree_record mid; superblock_get_tree_record(&ctx, &mid); - ASSERT_EQUAL(TEST_LOG_L1.meta_addr, mid.sealed_log.meta_addr); - ASSERT_EQUAL(TEST_LOG_L2.meta_addr, mid.live_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L1.head.meta_addr, mid.sealed_log.head.meta_addr); + ASSERT_EQUAL(TEST_LOG_L2.head.meta_addr, mid.live_log.head.meta_addr); // Commit a root that stops short of L1's last generation: L1 must be kept. superblock_snapshot_tree(&ctx, 0x4000, 5); @@ -261,8 +259,8 @@ CTEST2(superblock, test_snapshot_preserves_unincorporated_sealed_log) superblock_tree_record got; superblock_get_tree_record(&ctx, &got); ASSERT_EQUAL(0x4000, got.root_addr); - ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); - ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.sealed_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L2.head.meta_addr, got.live_log.head.meta_addr); + ASSERT_EQUAL(TEST_LOG_L1.head.meta_addr, got.sealed_log.head.meta_addr); ASSERT_EQUAL(TEST_LOG_L1.start_generation, got.sealed_log.start_generation); // Now a root that covers all of L1's generations: it is dropped. @@ -317,7 +315,7 @@ CTEST2(superblock, test_two_log_checkpoint_torn_begin) superblock_tree_record got; superblock_get_tree_record(&ctx, &got); ASSERT_EQUAL(0x4000, got.root_addr); - ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.live_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L1.head.meta_addr, got.live_log.head.meta_addr); ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); superblock_context_deinit(&ctx); } diff --git a/tests/unit/writeback_set_test.c b/tests/unit/writeback_set_test.c new file mode 100644 index 00000000..a8d60d1b --- /dev/null +++ b/tests/unit/writeback_set_test.c @@ -0,0 +1,291 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * writeback_set_test.c -- + * + * Unit tests for the writeback set (src/writeback_set.c). + * + * These drive a bare clockcache: allocate pages, dirty them, and check that + * a set issues their writes, waits for completion, and makes them durable. + * The observable that matters is cache_count_dirty() -- if + * writeback_set_wait() returned before the writes landed, pages would still + * be dirty afterwards. + */ + +#include "unit_tests.h" +#include "ctest.h" // This is required for all test-case files. + +#include "functional/test.h" +#include "splinterdb/data.h" +#include "../config.h" +#include "platform_io.h" +#include "platform_units.h" +#include "rc_allocator.h" +#include "clockcache.h" +// For the init_*_config_from_master_config() helpers. +#include "btree_test_common.h" +#include "writeback_set.h" +#include "poison.h" + +CTEST_DATA(writeback_set) +{ + master_config master_cfg; + data_config *data_cfg; + io_config io_cfg; + allocator_config allocator_cfg; + clockcache_config cache_cfg; + + platform_heap_id hid; + io_handle *io; + rc_allocator al; + clockcache cc; +}; + +CTEST_SETUP(writeback_set) +{ + platform_register_thread(); + config_set_defaults(&data->master_cfg); + data->master_cfg.cache_capacity = MiB_TO_B(64); + data->data_cfg = test_data_config; + + if (!SUCCESS( + config_parse(&data->master_cfg, 1, Ctest_argc, (char **)Ctest_argv)) + || !init_data_config_from_master_config(data->data_cfg, + &data->master_cfg) + || !init_io_config_from_master_config(&data->io_cfg, &data->master_cfg) + || !init_rc_allocator_config_from_master_config( + &data->allocator_cfg, &data->master_cfg, &data->io_cfg) + || !init_clockcache_config_from_master_config( + &data->cache_cfg, &data->master_cfg, &data->io_cfg)) + { + ASSERT_TRUE(FALSE, "Failed to parse args\n"); + } + + if (!SUCCESS(platform_heap_create(platform_get_module_id(), + 512 * MiB, + data->master_cfg.use_shmem, + &data->hid))) + { + ASSERT_TRUE(FALSE, "Failed to init heap\n"); + } + + data->io = io_handle_create(&data->io_cfg, data->hid); + ASSERT_NOT_NULL(data->io); + + ASSERT_TRUE(SUCCESS(rc_allocator_init(&data->al, + &data->allocator_cfg, + data->io, + data->hid, + platform_get_module_id()))); + ASSERT_TRUE(SUCCESS(clockcache_init(&data->cc, + &data->cache_cfg, + data->io, + (allocator *)&data->al, + "test", + data->hid, + platform_get_module_id()))); +} + +CTEST_TEARDOWN(writeback_set) +{ + clockcache_deinit(&data->cc); + rc_allocator_deinit(&data->al); + io_handle_destroy(data->io); + platform_heap_destroy(&data->hid); + platform_deregister_thread(); +} + +/* + * One extent, deliberately. Issuing many writes before waiting is *less* + * sensitive, not more: each submission to the io layer tends to reap earlier + * completions, so a large set cleans itself during the issue phase and a + * writeback_set_wait() that never waited would still look correct. Keeping the + * issue phase short leaves the completions genuinely outstanding when wait() is + * called. + */ + +/* + * Allocate one extent's worth of pages and dirty every one of them. Returns the + * extent's base address; page i is at base + i * page_size. + */ +static uint64 +alloc_and_dirty_extent(clockcache *cc, rc_allocator *al) +{ + uint64 base_addr; + platform_status rc = + allocator_alloc((allocator *)al, &base_addr, PAGE_TYPE_MISC); + ASSERT_TRUE(SUCCESS(rc)); + + cache *ccp = (cache *)cc; + uint64 page_size = cache_config_page_size(cache_get_config(ccp)); + uint64 pages_per_extent = + cache_config_pages_per_extent(cache_get_config(ccp)); + + for (uint64 i = 0; i < pages_per_extent; i++) { + uint64 addr = base_addr + i * page_size; + page_handle *page = cache_alloc(ccp, addr, PAGE_TYPE_MISC); + ASSERT_NOT_NULL(page); + // cache_alloc hands back a dirty, write-locked page. + memset(page->data, (int)(i & 0xff), page_size); + cache_unlock(ccp, page); + cache_unclaim(ccp, page); + cache_unget(ccp, page); + } + return base_addr; +} + +/* + * A set of individually added pages is issued, waited for, and made durable, + * and the pages really are clean afterwards. + */ +CTEST2(writeback_set, test_wait_makes_pages_clean) +{ + cache *ccp = (cache *)&data->cc; + uint64 page_size = cache_config_page_size(cache_get_config(ccp)); + uint64 num_pages = cache_config_pages_per_extent(cache_get_config(ccp)); + + uint64 base_addr = alloc_and_dirty_extent(&data->cc, &data->al); + ASSERT_EQUAL(num_pages, cache_count_dirty(ccp)); + + writeback_set set; + writeback_set_init(&set, ccp, data->hid); + + for (uint64 i = 0; i < num_pages; i++) { + uint64 addr = base_addr + i * page_size; + page_handle *page = cache_get(ccp, addr, TRUE, PAGE_TYPE_MISC); + ASSERT_NOT_NULL(page); + ASSERT_TRUE(SUCCESS(writeback_set_add_page(&set, page, PAGE_TYPE_MISC))); + cache_unget(ccp, page); + } + ASSERT_EQUAL(num_pages, writeback_set_num_requests(&set)); + + ASSERT_TRUE(SUCCESS(writeback_set_wait(&set))); + /* + * Checked before make_durable(), deliberately: wait() is what establishes + * completion, and letting the barrier run first would give the I/O extra + * time to land and mask a wait() that did not actually wait. + */ + ASSERT_EQUAL(0, cache_count_dirty(ccp)); + + ASSERT_TRUE(SUCCESS(writeback_set_make_durable(&set))); + writeback_set_deinit(&set); +} + +/* The same, but adding the extent in one call rather than page by page. */ +CTEST2(writeback_set, test_add_extent) +{ + cache *ccp = (cache *)&data->cc; + + uint64 base_addr = alloc_and_dirty_extent(&data->cc, &data->al); + ASSERT_NOT_EQUAL(0, cache_count_dirty(ccp)); + + writeback_set set; + writeback_set_init(&set, ccp, data->hid); + + ASSERT_TRUE( + SUCCESS(writeback_set_add_extent(&set, base_addr, PAGE_TYPE_MISC))); + // One request covers the whole extent, however many pages it holds. + ASSERT_EQUAL(1, writeback_set_num_requests(&set)); + + ASSERT_TRUE(SUCCESS(writeback_set_wait(&set))); + ASSERT_EQUAL(0, cache_count_dirty(ccp)); // before the barrier; see above + + ASSERT_TRUE(SUCCESS(writeback_set_make_durable(&set))); + writeback_set_deinit(&set); +} + +/* Waiting on an empty set is legal and does nothing. */ +CTEST2(writeback_set, test_empty_set) +{ + cache *ccp = (cache *)&data->cc; + writeback_set set; + writeback_set_init(&set, ccp, data->hid); + + ASSERT_EQUAL(0, writeback_set_num_requests(&set)); + ASSERT_TRUE(SUCCESS(writeback_set_wait(&set))); + ASSERT_TRUE(SUCCESS(writeback_set_make_durable(&set))); + + writeback_set_deinit(&set); +} + +/* + * Adding an already-clean page is a no-op the set still accounts for: the + * request carries gen == 0, and waiting on it completes immediately. + */ +CTEST2(writeback_set, test_add_clean_page) +{ + cache *ccp = (cache *)&data->cc; + uint64 page_size = cache_config_page_size(cache_get_config(ccp)); + uint64 num_pages = cache_config_pages_per_extent(cache_get_config(ccp)); + + uint64 base_addr = alloc_and_dirty_extent(&data->cc, &data->al); + + // Clean everything first, so the pages below are already written back. + writeback_set flush; + writeback_set_init(&flush, ccp, data->hid); + ASSERT_TRUE( + SUCCESS(writeback_set_add_extent(&flush, base_addr, PAGE_TYPE_MISC))); + ASSERT_TRUE(SUCCESS(writeback_set_wait(&flush))); + writeback_set_deinit(&flush); + ASSERT_EQUAL(0, cache_count_dirty(ccp)); + + writeback_set set; + writeback_set_init(&set, ccp, data->hid); + for (uint64 i = 0; i < num_pages; i++) { + uint64 addr = base_addr + i * page_size; + page_handle *page = cache_get(ccp, addr, TRUE, PAGE_TYPE_MISC); + ASSERT_NOT_NULL(page); + ASSERT_TRUE(SUCCESS(writeback_set_add_page(&set, page, PAGE_TYPE_MISC))); + cache_unget(ccp, page); + } + + ASSERT_TRUE(SUCCESS(writeback_set_wait(&set))); + ASSERT_EQUAL(0, cache_count_dirty(ccp)); // before the barrier; see above + + ASSERT_TRUE(SUCCESS(writeback_set_make_durable(&set))); + writeback_set_deinit(&set); +} + +/* + * A set may be reused across rounds: dirty, flush, dirty again, flush again. + * Catches a wait() that trusted stale receipts from an earlier round. + */ +CTEST2(writeback_set, test_repeated_rounds) +{ + cache *ccp = (cache *)&data->cc; + uint64 page_size = cache_config_page_size(cache_get_config(ccp)); + uint64 num_pages = cache_config_pages_per_extent(cache_get_config(ccp)); + + uint64 base_addr = alloc_and_dirty_extent(&data->cc, &data->al); + + for (uint64 round = 0; round < 4; round++) { + if (round > 0) { + // Re-dirty every page. + for (uint64 i = 0; i < num_pages; i++) { + uint64 addr = base_addr + i * page_size; + page_handle *page = cache_get(ccp, addr, TRUE, PAGE_TYPE_MISC); + ASSERT_NOT_NULL(page); + while (!cache_try_claim(ccp, page)) { + cache_unget(ccp, page); + page = cache_get(ccp, addr, TRUE, PAGE_TYPE_MISC); + } + cache_lock(ccp, page); + memset(page->data, (int)(round & 0xff), page_size); + cache_unlock(ccp, page); + cache_unclaim(ccp, page); + cache_unget(ccp, page); + } + ASSERT_NOT_EQUAL(0, cache_count_dirty(ccp)); + } + + writeback_set set; + writeback_set_init(&set, ccp, data->hid); + ASSERT_TRUE( + SUCCESS(writeback_set_add_extent(&set, base_addr, PAGE_TYPE_MISC))); + ASSERT_TRUE(SUCCESS(writeback_set_wait(&set))); + ASSERT_EQUAL(0, cache_count_dirty(ccp)); // before the barrier; see above + ASSERT_TRUE(SUCCESS(writeback_set_make_durable(&set))); + writeback_set_deinit(&set); + } +}