From b250514e863bf464b2e5269907cac263b93709e9 Mon Sep 17 00:00:00 2001 From: Ben Reisner Date: Thu, 30 Jul 2026 13:18:25 -0700 Subject: [PATCH] Freelist Arena blocks to consume wasted memory When an allocation request exceeds the remaining capacity of the current Arena block, put the remainder on a freelist (if >= sizeof(Block) after alignment) instead of discarding it. Before allocating fresh memory from system_allocator, search the freelist for a block capable of satisfying the allocation. PiperOrigin-RevId: 956699564 --- tcmalloc/arena.cc | 133 +++++++++++++++++------ tcmalloc/arena.h | 25 ++++- tcmalloc/arena_test.cc | 225 ++++++++++++++++++++++++++++++++++++++- tcmalloc/global_stats.cc | 7 +- 4 files changed, 350 insertions(+), 40 deletions(-) diff --git a/tcmalloc/arena.cc b/tcmalloc/arena.cc index a6fd93108..8698bf3c8 100644 --- a/tcmalloc/arena.cc +++ b/tcmalloc/arena.cc @@ -27,55 +27,122 @@ #include "tcmalloc/internal/memory_tag.h" #include "tcmalloc/internal/system_allocator.h" #include "tcmalloc/parameters.h" +#include "tcmalloc/span.h" #include "tcmalloc/static_vars.h" GOOGLE_MALLOC_SECTION_BEGIN namespace tcmalloc { namespace tcmalloc_internal { -void* Arena::Alloc(size_t bytes, std::align_val_t alignment) { - size_t align = static_cast(alignment); - TC_ASSERT_GT(align, 0); +namespace { +inline size_t AlignmentBytes(char* area, size_t align) { + const size_t misalignment = reinterpret_cast(area) % align; + return misalignment != 0 ? align - misalignment : 0; +} +} // namespace - AllocationGuardSpinLockHolder l(arena_lock_); +void Arena::StashRemainingFreeArea() { + size_t block_align_bytes = AlignmentBytes(free_area_, alignof(Block)); + // If the remaining free area is large enough to hold a block AND allocate a + // span (smallest allocation served via arena), add it to the freelist. + // Otherwise, mark it as unavailable and drop it. + if (freelist_blocks_ < kMaxFreelistBlocks && + free_avail_ >= + block_align_bytes + std::max(sizeof(Span), sizeof(Block))) { + bytes_unavailable_ += block_align_bytes; + Block* b = reinterpret_cast(free_area_ + block_align_bytes); + b->next = freelist_; + b->size = free_avail_ - block_align_bytes; + freelist_ = b; + freelist_blocks_++; + freelist_bytes_unallocated_ += b->size; + } else { + bytes_unavailable_ += free_avail_; + } + free_area_ = nullptr; + free_avail_ = 0; +} + +Arena::Block* Arena::TryPopFromFreelist(size_t bytes, size_t align) { + Block* cur = freelist_; + Block* prev = nullptr; + while (cur != nullptr) { + char* area = reinterpret_cast(cur); + size_t avail = cur->size; + size_t align_bytes = AlignmentBytes(area, align); + if (avail < align_bytes + bytes) { + prev = cur; + cur = cur->next; + continue; + } + // Found a block on the freelist that can satisfy the request. Remove it + // from the freelist and return it. + if (prev == nullptr) { + freelist_ = cur->next; + } else { + prev->next = cur->next; + } + freelist_blocks_--; + freelist_bytes_unallocated_ -= cur->size; + return cur; + } + return nullptr; +} + +void Arena::AllocSlow(size_t bytes, size_t align) { + // Make a new block on the freelist from any remaining free area. + StashRemainingFreeArea(); - { // First we need to move up to the correct alignment. - const int misalignment = reinterpret_cast(free_area_) % align; - const int alignment_bytes = misalignment != 0 ? align - misalignment : 0; - free_area_ += alignment_bytes; - free_avail_ -= alignment_bytes; - bytes_allocated_ += alignment_bytes; + // Try to use a block from the freelist. + if (Block* b = TryPopFromFreelist(bytes, align); b != nullptr) { + free_area_ = reinterpret_cast(b); + free_avail_ = b->size; + return; } - char* result; + + // If no block on the freelist can satisfy the request, allocate a new block + // from the system allocator. auto& system_allocator = tc_globals.system_allocator(); - if (free_avail_ < bytes) { - size_t ask = bytes > kAllocIncrement ? bytes : kAllocIncrement; - auto [ptr, actual_size] = system_allocator.Allocate( - ask, std::max(kPageSize, align), MemoryTag::kMetadata); - free_area_ = reinterpret_cast(ptr); - if (ABSL_PREDICT_FALSE(free_area_ == nullptr)) { - TC_BUG( - "FATAL ERROR: Out of memory trying to allocate internal tcmalloc " - "data (bytes=%v, object-size=%v); is something preventing mmap from " - "succeeding (sandbox, VSS limitations)?", - kAllocIncrement, bytes); - } + size_t ask = bytes > kAllocIncrement ? bytes : kAllocIncrement; + auto [ptr, actual_size] = system_allocator.Allocate( + ask, std::max(kPageSize, align), MemoryTag::kMetadata); + free_area_ = reinterpret_cast(ptr); + if (ABSL_PREDICT_FALSE(free_area_ == nullptr)) { + TC_BUG( + "FATAL ERROR: Out of memory trying to allocate internal tcmalloc " + "data (bytes=%v, object-size=%v); is something preventing mmap from " + "succeeding (sandbox, VSS limitations)?", + kAllocIncrement, bytes); + } - if (Parameters::back_small_allocations() && - actual_size <= Parameters::back_size_threshold_bytes()) { - system_allocator.Back(free_area_, actual_size); - } + if (Parameters::back_small_allocations() && + actual_size <= Parameters::back_size_threshold_bytes()) { + system_allocator.Back(free_area_, actual_size); + } - // We've discarded the previous free_area_, so any bytes that were - // unallocated are effectively inaccessible to future allocations. - bytes_unavailable_ += free_avail_; - blocks_++; + blocks_++; + free_avail_ = actual_size; +} - free_avail_ = actual_size; +void* Arena::Alloc(size_t bytes, std::align_val_t alignment) { + size_t align = static_cast(alignment); + TC_ASSERT_GT(align, 0); + + AllocationGuardSpinLockHolder l(arena_lock_); + + size_t alignment_bytes = AlignmentBytes(free_area_, align); + if (ABSL_PREDICT_FALSE(free_avail_ < alignment_bytes + bytes)) { + AllocSlow(bytes, align); + alignment_bytes = AlignmentBytes(free_area_, align); } + TC_ASSERT_GE(free_avail_, alignment_bytes + bytes); + free_area_ += alignment_bytes; + free_avail_ -= alignment_bytes; + bytes_allocated_ += alignment_bytes; + TC_ASSERT_EQ(reinterpret_cast(free_area_) % align, 0); - result = free_area_; + char* result = free_area_; free_area_ += bytes; free_avail_ -= bytes; bytes_allocated_ += bytes; diff --git a/tcmalloc/arena.h b/tcmalloc/arena.h index d86eb0cd3..8f950dbaf 100644 --- a/tcmalloc/arena.h +++ b/tcmalloc/arena.h @@ -46,6 +46,8 @@ struct ArenaStats { // The number of blocks allocated by the Arena. size_t blocks; + // The number of blocks currently on the freelist. + size_t freelist_blocks; }; // Arena allocation; designed for use by tcmalloc internal data structures like @@ -76,7 +78,8 @@ class ABSL_CACHELINE_ALIGNED Arena { ArenaStats s; s.bytes_allocated = bytes_allocated_; - s.bytes_unallocated = free_avail_; + s.bytes_unallocated = free_avail_ + freelist_bytes_unallocated_; + s.freelist_blocks = freelist_blocks_; s.bytes_unavailable = bytes_unavailable_; s.bytes_nonresident = bytes_nonresident_; s.blocks = blocks_; @@ -84,8 +87,21 @@ class ABSL_CACHELINE_ALIGNED Arena { } private: + struct Block { + Block* next; + size_t size; + }; + + ABSL_ATTRIBUTE_NOINLINE void AllocSlow(size_t bytes, size_t align) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(arena_lock_); + void StashRemainingFreeArea() ABSL_EXCLUSIVE_LOCKS_REQUIRED(arena_lock_); + Block* TryPopFromFreelist(size_t bytes, size_t align) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(arena_lock_); + // How much to allocate from system at a time static constexpr int kAllocIncrement = 128 << 10; + // Maximum number of blocks to keep on freelist_. + static constexpr int kMaxFreelistBlocks = 100; mutable absl::base_internal::SpinLock arena_lock_{ absl::base_internal::SCHEDULE_KERNEL_ONLY}; @@ -93,6 +109,7 @@ class ABSL_CACHELINE_ALIGNED Arena { // Free area from which to carve new objects char* free_area_ ABSL_GUARDED_BY(arena_lock_) = nullptr; size_t free_avail_ ABSL_GUARDED_BY(arena_lock_) = 0; + Block* freelist_ ABSL_GUARDED_BY(arena_lock_) = nullptr; // Total number of bytes allocated from this arena size_t bytes_allocated_ ABSL_GUARDED_BY(arena_lock_) = 0; @@ -103,7 +120,11 @@ class ABSL_CACHELINE_ALIGNED Arena { // that these bytes are disjoint from the ones counted in `bytes_allocated`. size_t bytes_nonresident_ ABSL_GUARDED_BY(arena_lock_) = 0; // Total number of blocks/free areas managed by this Arena. - size_t blocks_ ABSL_GUARDED_BY(arena_lock_) = 0; + uint16_t blocks_ ABSL_GUARDED_BY(arena_lock_) = 0; + // Total number of blocks on the freelist. Capped at kMaxFreelistBlocks. + uint8_t freelist_blocks_ ABSL_GUARDED_BY(arena_lock_) = 0; + // Total number of bytes on the freelist. + uint32_t freelist_bytes_unallocated_ ABSL_GUARDED_BY(arena_lock_) = 0; Arena(const Arena&) = delete; Arena& operator=(const Arena&) = delete; diff --git a/tcmalloc/arena_test.cc b/tcmalloc/arena_test.cc index d677e893f..285c5af0b 100644 --- a/tcmalloc/arena_test.cc +++ b/tcmalloc/arena_test.cc @@ -16,6 +16,7 @@ #include +#include #include #include "gtest/gtest.h" @@ -73,10 +74,12 @@ TEST(Arena, Stats) { EXPECT_EQ(stats_after_alloc.bytes_unavailable, 0); EXPECT_EQ(stats_after_alloc.bytes_nonresident, 0); EXPECT_EQ(stats_after_alloc.blocks, 1); + EXPECT_EQ(stats_after_alloc.freelist_blocks, 0); // Trigger an allocation that is larger than the remaining free bytes. // - // TODO(b/201694482): Optimize this. + // b/201694482: The remaining bytes from the first block are freelisted rather + // than being wasted, with only 7 bytes lost to alignment padding for Block. ptr = arena.Alloc(stats_after_alloc.bytes_unallocated + 1, Align(1)); ArenaStats stats_after_alloc2 = arena.stats(); EXPECT_NE(ptr, nullptr); @@ -84,10 +87,10 @@ TEST(Arena, Stats) { EXPECT_EQ(stats_after_alloc2.bytes_allocated, stats_after_alloc.bytes_unallocated + 2); EXPECT_GE(stats_after_alloc2.bytes_unallocated, 0); - EXPECT_EQ(stats_after_alloc2.bytes_unavailable, - stats_after_alloc.bytes_unallocated); + EXPECT_EQ(stats_after_alloc2.bytes_unavailable, 7); EXPECT_EQ(stats_after_alloc.bytes_nonresident, 0); EXPECT_EQ(stats_after_alloc2.blocks, 2); + EXPECT_EQ(stats_after_alloc2.freelist_blocks, 1); } TEST(Arena, ReportUnmapped) { @@ -131,6 +134,222 @@ TEST(Arena, BytesImpending) { EXPECT_EQ(stats.bytes_allocated, 100); } +TEST(Arena, FreelistReuse) { + Arena arena; + // Step 1: Allocate from initial arena block and record remaining capacity. + void* p1 = arena.Alloc(104, Align(8)); + EXPECT_NE(p1, nullptr); + ArenaStats s1 = arena.stats(); + EXPECT_EQ(s1.blocks, 1); + EXPECT_EQ(s1.bytes_allocated, 104); + size_t b1_remaining = s1.bytes_unallocated; + + // Step 2: Request more bytes than remain in block 1. + // This pushes block 1's remainder onto freelist_ and allocates block 2 from + // the OS. + void* p2 = arena.Alloc(b1_remaining + 104, Align(8)); + EXPECT_NE(p2, nullptr); + ArenaStats s2 = arena.stats(); + EXPECT_EQ(s2.blocks, 2); + EXPECT_EQ(s2.bytes_unavailable, 0); + + // Step 3: Drain block 2 down to 100 bytes so it cannot satisfy a 1000-byte + // request. The if check protects against underflow in case block 2 is already + // <= 100 bytes. + size_t b2_remaining = s2.bytes_unallocated > b1_remaining + ? s2.bytes_unallocated - b1_remaining + : 0; + if (b2_remaining > 100) { + void* p_drain = arena.Alloc(b2_remaining - 100, Align(8)); + EXPECT_NE(p_drain, nullptr); + } + + // Step 4: Request 1000 bytes. Block 2 cannot satisfy this (only 100 bytes + // left), but block 1 on freelist_ can. Arena should reuse block 1 without + // allocating block 3. + void* p3 = arena.Alloc(1000, Align(8)); + EXPECT_NE(p3, nullptr); + ArenaStats s3 = arena.stats(); + EXPECT_EQ(s3.blocks, 2); + EXPECT_EQ(s3.bytes_unavailable, 0); +} + +TEST(Arena, FreelistMultipleBlocks) { + Arena arena; + // Step 1: Allocate from block 1, then drain it down to 5000 bytes remaining. + void* p1 = arena.Alloc(128, Align(8)); + EXPECT_NE(p1, nullptr); + ArenaStats s1 = arena.stats(); + size_t to_drain = + s1.bytes_unallocated > 5000 ? s1.bytes_unallocated - 5000 : 0; + if (to_drain > 0) { + void* p_drain1 = arena.Alloc(to_drain, Align(8)); + EXPECT_NE(p_drain1, nullptr); + } + + // Step 2: Request 10000 bytes. Block 1 (5000 bytes left) is pushed to + // freelist_, and block 2 is allocated from the OS. + void* p2 = arena.Alloc(10000, Align(8)); + EXPECT_NE(p2, nullptr); + ArenaStats s2 = arena.stats(); + EXPECT_EQ(s2.blocks, 2); + EXPECT_EQ(s2.bytes_unavailable, 0); + + // Step 3: Drain block 2 down to 3000 bytes remaining. + size_t b2_avail = + s2.bytes_unallocated > 5000 ? s2.bytes_unallocated - 5000 : 0; + if (b2_avail > 3000) { + void* p_drain2 = arena.Alloc(b2_avail - 3000, Align(8)); + EXPECT_NE(p_drain2, nullptr); + } + + // Step 4: Request 10000 bytes. Neither active block 2 (3000 left) nor + // freelisted block 1 (5000 left) can satisfy this. Block 2 is pushed to + // freelist_, and block 3 is allocated from the OS. + void* p3 = arena.Alloc(10000, Align(8)); + EXPECT_NE(p3, nullptr); + ArenaStats s3 = arena.stats(); + EXPECT_EQ(s3.blocks, 3); + EXPECT_EQ(s3.bytes_unavailable, 0); + + // Step 5: Drain active block 3 down to 1000 bytes remaining. + size_t b3_avail = s3.bytes_unallocated > 5000 + 3000 + ? s3.bytes_unallocated - 5000 - 3000 + : 0; + if (b3_avail > 1000) { + void* p_drain3 = arena.Alloc(b3_avail - 1000, Align(8)); + EXPECT_NE(p_drain3, nullptr); + } + + // Step 6: Request 2000 bytes. Active block 3 (1000 left) cannot satisfy + // this, but block 2 on freelist_ (3000 left) can. It is reused without + // adding a new OS block. + void* p4 = arena.Alloc(2000, Align(8)); + EXPECT_NE(p4, nullptr); + ArenaStats s4 = arena.stats(); + EXPECT_EQ(s4.blocks, 3); + EXPECT_EQ(s4.bytes_unavailable, 0); + + // Step 7: Request 4000 bytes. Block 1 on freelist_ (5000 left) satisfies + // this. + void* p5 = arena.Alloc(4000, Align(8)); + EXPECT_NE(p5, nullptr); + ArenaStats s5 = arena.stats(); + EXPECT_EQ(s5.blocks, 3); + EXPECT_EQ(s5.bytes_unavailable, 0); +} + +TEST(Arena, SmallRemainderUnavailable) { + Arena arena; + // Step 1: Allocate 1 byte to initialize the first OS block and record its + // capacity. + void* p1 = arena.Alloc(1, Align(1)); + EXPECT_NE(p1, nullptr); + ArenaStats s1 = arena.stats(); + size_t rem = s1.bytes_unallocated; + EXPECT_EQ(s1.freelist_blocks, 0); + EXPECT_EQ(s1.bytes_unavailable, 0); + EXPECT_EQ(s1.freelist_blocks, 0); + + // Step 2: Consume all but 4 bytes of the first block. + void* p2 = arena.Alloc(rem - 4, Align(1)); + EXPECT_NE(p2, nullptr); + ArenaStats s2 = arena.stats(); + EXPECT_EQ(s2.freelist_blocks, 0); + EXPECT_EQ(s2.bytes_unallocated, 4); + EXPECT_EQ(s2.bytes_unavailable, 0); + EXPECT_EQ(s2.freelist_blocks, 0); + + // Step 3: Request 100 bytes. The remaining 4 bytes cannot satisfy the request + // and are too small (< sizeof(Block) == 16) to be placed on freelist_. They + // are marked as bytes_unavailable, and block 2 is allocated from the OS. + void* p3 = arena.Alloc(100, Align(1)); + ArenaStats s3 = arena.stats(); + EXPECT_EQ(s3.bytes_unavailable, 4); + EXPECT_EQ(s3.blocks, 2); + EXPECT_EQ(s3.freelist_blocks, 0); + EXPECT_NE(p3, nullptr); +} + +TEST(Arena, FreelistAlignmentEdgeCase) { + Arena arena; + // Step 1: Initialize block 1 and record its remaining capacity. + void* p1 = arena.Alloc(1, Align(1)); + EXPECT_NE(p1, nullptr); + ArenaStats s1 = arena.stats(); + size_t rem = s1.bytes_unallocated; + EXPECT_GT(rem, 200); + + // Step 2: Leave exactly 134 bytes remaining in block 1. + void* p2 = arena.Alloc(rem - 134, Align(1)); + EXPECT_NE(p2, nullptr); + ArenaStats s2 = arena.stats(); + EXPECT_EQ(s2.bytes_unallocated, 134); + + // Step 3: Request 2000 bytes. Block 1 (134 bytes left) is placed on + // freelist_, and block 2 is allocated from the OS. + void* p3 = arena.Alloc(2000, Align(64)); + EXPECT_NE(p3, nullptr); + ArenaStats s3 = arena.stats(); + EXPECT_EQ(s3.blocks, 2); + + // Step 4: Drain active block 2 down to 10 bytes remaining. + size_t b2_rem = s3.bytes_unallocated > 134 ? s3.bytes_unallocated - 134 : 0; + if (b2_rem > 10) { + void* p_drain = arena.Alloc(b2_rem - 10, Align(1)); + EXPECT_NE(p_drain, nullptr); + } + + // Step 5: Request 150 bytes with 64-byte alignment. + // Although block 1 on freelist_ has 134 bytes, satisfying a 150-byte request + // exceeds 134 bytes. Thus block 1 cannot be used, and block 3 is allocated + // from the OS. + void* p4 = arena.Alloc(150, Align(64)); + EXPECT_NE(p4, nullptr); + ArenaStats s4 = arena.stats(); + EXPECT_EQ(s4.blocks, 3); + + // Step 6: Request 80 bytes with 4-byte alignment. + // With smaller size and alignment requirements, block 1 on freelist_ (134 + // bytes) can satisfy the request and is reused without allocating an OS + // block. + void* p5 = arena.Alloc(80, Align(4)); + EXPECT_NE(p5, nullptr); + ArenaStats s5 = arena.stats(); + EXPECT_EQ(s5.blocks, 3); +} + +TEST(Arena, FreelistLimit) { + Arena dummy_arena; + dummy_arena.Alloc(1, Align(1)); + size_t block_size = dummy_arena.stats().bytes_unallocated + 1; + + // We want to force the freelist to grow beyond 100 blocks. + // By default, the freelist limit is 100. + // We allocate 2/3 of a block on each call. Because the remaining 1/3 of the + // block is less than the requested 2/3, each new allocation will force the + // remainder of the active block to be stashed on the freelist and a new block + // to be allocated. Because the stashed blocks (size 1/3) are smaller than the + // request size (2/3), they can never be reused, causing the freelist to grow + // monotonically. Only the first 100 blocks should be kept on the freelist. + // We align X to 8 bytes to avoid alignment padding during stashing. + size_t X = ((block_size * 2) / 3) & ~7; + + Arena arena; + for (int i = 0; i < 105; ++i) { + void* p = arena.Alloc(X, Align(1)); + EXPECT_NE(p, nullptr); + } + + ArenaStats stats = arena.stats(); + EXPECT_EQ(stats.freelist_blocks, 100); + + size_t expected_unallocated = 101 * (block_size - X); + EXPECT_EQ(stats.bytes_unallocated, expected_unallocated); + + EXPECT_EQ(stats.bytes_allocated, X * 105); +} + } // namespace } // namespace tcmalloc_internal } // namespace tcmalloc diff --git a/tcmalloc/global_stats.cc b/tcmalloc/global_stats.cc index f7f8be821..7d53e4219 100644 --- a/tcmalloc/global_stats.cc +++ b/tcmalloc/global_stats.cc @@ -376,7 +376,8 @@ void DumpStats(Printer& out, int level) { "MALLOC: %12u Tcmalloc page size\n" "MALLOC: %12u Tcmalloc hugepage size\n" "MALLOC: %12u CPUs Allowed in Mask\n" - "MALLOC: %12u Arena blocks\n", + "MALLOC: %12u Arena blocks\n" + "MALLOC: %12u Arena freelist blocks\n", bytes_in_use_by_app, bytes_in_use_by_app / MiB, stats.pageheap.free_bytes, stats.pageheap.free_bytes / MiB, stats.central_bytes, stats.central_bytes / MiB, @@ -417,7 +418,8 @@ void DumpStats(Printer& out, int level) { uint64_t(kPageSize), uint64_t(kHugePageSize), CountAllowedCpus(), - stats.arena.blocks + stats.arena.blocks, + stats.arena.freelist_blocks ); // clang-format on @@ -774,6 +776,7 @@ void DumpStatsInPbtxt(Printer& out, int level) { region.PrintI64("tcmalloc_huge_page_size", uint64_t(kHugePageSize)); region.PrintI64("cpus_allowed", CountAllowedCpus()); region.PrintI64("arena_blocks", stats.arena.blocks); + region.PrintI64("arena_freelist_blocks", stats.arena.freelist_blocks); // Print hooks stats. region.PrintI64("new_hooks_present", uint64_t(new_hooks_.size()));