Freelist Arena blocks to consume wasted memory

tcmalloc/arena.cc

@@ -27,55 +27,122 @@
 #include "tcmalloc/internal/memory_tag.h"
 #include "tcmalloc/internal/system_allocator.h"
 #include "tcmalloc/parameters.h"
+#include "tcmalloc/span.h"
 #include "tcmalloc/static_vars.h"
 
 GOOGLE_MALLOC_SECTION_BEGIN
 namespace tcmalloc {
 namespace tcmalloc_internal {
 
-void* Arena::Alloc(size_t bytes, std::align_val_t alignment) {
-  size_t align = static_cast<size_t>(alignment);
-  TC_ASSERT_GT(align, 0);
+namespace {
+inline size_t AlignmentBytes(char* area, size_t align) {
+  const size_t misalignment = reinterpret_cast<uintptr_t>(area) % align;
+  return misalignment != 0 ? align - misalignment : 0;
+}
+}  // namespace
 
-  AllocationGuardSpinLockHolder l(arena_lock_);
+void Arena::StashRemainingFreeArea() {
+  size_t block_align_bytes = AlignmentBytes(free_area_, alignof(Block));
+  // If the remaining free area is large enough to hold a block AND allocate a
+  // span (smallest allocation served via arena), add it to the freelist.
+  // Otherwise, mark it as unavailable and drop it.
+  if (freelist_blocks_ < kMaxFreelistBlocks &&
+      free_avail_ >=
+          block_align_bytes + std::max(sizeof(Span), sizeof(Block))) {
+    bytes_unavailable_ += block_align_bytes;
+    Block* b = reinterpret_cast<Block*>(free_area_ + block_align_bytes);
+    b->next = freelist_;
+    b->size = free_avail_ - block_align_bytes;
+    freelist_ = b;
+    freelist_blocks_++;
+    freelist_bytes_unallocated_ += b->size;
+  } else {
+    bytes_unavailable_ += free_avail_;
+  }
+  free_area_ = nullptr;
+  free_avail_ = 0;
+}
+
+Arena::Block* Arena::TryPopFromFreelist(size_t bytes, size_t align) {
+  Block* cur = freelist_;
+  Block* prev = nullptr;
+  while (cur != nullptr) {
+    char* area = reinterpret_cast<char*>(cur);
+    size_t avail = cur->size;
+    size_t align_bytes = AlignmentBytes(area, align);
+    if (avail < align_bytes + bytes) {
+      prev = cur;
+      cur = cur->next;
+      continue;
+    }
+    // Found a block on the freelist that can satisfy the request. Remove it
+    // from the freelist and return it.
+    if (prev == nullptr) {
+      freelist_ = cur->next;
+    } else {
+      prev->next = cur->next;
+    }
+    freelist_blocks_--;
+    freelist_bytes_unallocated_ -= cur->size;
+    return cur;
+  }
+  return nullptr;
+}
+
+void Arena::AllocSlow(size_t bytes, size_t align) {
+  // Make a new block on the freelist from any remaining free area.
+  StashRemainingFreeArea();
 
-  {  // First we need to move up to the correct alignment.
-    const int misalignment = reinterpret_cast<uintptr_t>(free_area_) % align;
-    const int alignment_bytes = misalignment != 0 ? align - misalignment : 0;
-    free_area_ += alignment_bytes;
-    free_avail_ -= alignment_bytes;
-    bytes_allocated_ += alignment_bytes;
+  // Try to use a block from the freelist.
+  if (Block* b = TryPopFromFreelist(bytes, align); b != nullptr) {
+    free_area_ = reinterpret_cast<char*>(b);
+    free_avail_ = b->size;
+    return;
   }
-  char* result;
+
+  // If no block on the freelist can satisfy the request, allocate a new block
+  // from the system allocator.
   auto& system_allocator = tc_globals.system_allocator();
-  if (free_avail_ < bytes) {
-    size_t ask = bytes > kAllocIncrement ? bytes : kAllocIncrement;
-    auto [ptr, actual_size] = system_allocator.Allocate(
-        ask, std::max(kPageSize, align), MemoryTag::kMetadata);
-    free_area_ = reinterpret_cast<char*>(ptr);
-    if (ABSL_PREDICT_FALSE(free_area_ == nullptr)) {
-      TC_BUG(
-          "FATAL ERROR: Out of memory trying to allocate internal tcmalloc "
-          "data (bytes=%v, object-size=%v); is something preventing mmap from "
-          "succeeding (sandbox, VSS limitations)?",
-          kAllocIncrement, bytes);
-    }
+  size_t ask = bytes > kAllocIncrement ? bytes : kAllocIncrement;
+  auto [ptr, actual_size] = system_allocator.Allocate(
+      ask, std::max(kPageSize, align), MemoryTag::kMetadata);
+  free_area_ = reinterpret_cast<char*>(ptr);
+  if (ABSL_PREDICT_FALSE(free_area_ == nullptr)) {
+    TC_BUG(
+        "FATAL ERROR: Out of memory trying to allocate internal tcmalloc "
+        "data (bytes=%v, object-size=%v); is something preventing mmap from "
+        "succeeding (sandbox, VSS limitations)?",
+        kAllocIncrement, bytes);
+  }
 
-    if (Parameters::back_small_allocations() &&
-        actual_size <= Parameters::back_size_threshold_bytes()) {
-      system_allocator.Back(free_area_, actual_size);
-    }
+  if (Parameters::back_small_allocations() &&
+      actual_size <= Parameters::back_size_threshold_bytes()) {
+    system_allocator.Back(free_area_, actual_size);
+  }
 
-    // We've discarded the previous free_area_, so any bytes that were
-    // unallocated are effectively inaccessible to future allocations.
-    bytes_unavailable_ += free_avail_;
-    blocks_++;
+  blocks_++;
+  free_avail_ = actual_size;
+}
 
-    free_avail_ = actual_size;
+void* Arena::Alloc(size_t bytes, std::align_val_t alignment) {
+  size_t align = static_cast<size_t>(alignment);
+  TC_ASSERT_GT(align, 0);
+
+  AllocationGuardSpinLockHolder l(arena_lock_);
+
+  size_t alignment_bytes = AlignmentBytes(free_area_, align);
+  if (ABSL_PREDICT_FALSE(free_avail_ < alignment_bytes + bytes)) {
+    AllocSlow(bytes, align);
+    alignment_bytes = AlignmentBytes(free_area_, align);
   }
 
+  TC_ASSERT_GE(free_avail_, alignment_bytes + bytes);
+  free_area_ += alignment_bytes;
+  free_avail_ -= alignment_bytes;
+  bytes_allocated_ += alignment_bytes;
+
   TC_ASSERT_EQ(reinterpret_cast<uintptr_t>(free_area_) % align, 0);
-  result = free_area_;
+  char* result = free_area_;
   free_area_ += bytes;
   free_avail_ -= bytes;
   bytes_allocated_ += bytes;

tcmalloc/arena.h

@@ -46,6 +46,8 @@ struct ArenaStats {
 
   // The number of blocks allocated by the Arena.
   size_t blocks;
+  // The number of blocks currently on the freelist.
+  size_t freelist_blocks;
 };
 
 // Arena allocation; designed for use by tcmalloc internal data structures like
@@ -76,23 +78,38 @@ class ABSL_CACHELINE_ALIGNED Arena {
 
     ArenaStats s;
     s.bytes_allocated = bytes_allocated_;
-    s.bytes_unallocated = free_avail_;
+    s.bytes_unallocated = free_avail_ + freelist_bytes_unallocated_;
+    s.freelist_blocks = freelist_blocks_;
     s.bytes_unavailable = bytes_unavailable_;
     s.bytes_nonresident = bytes_nonresident_;
     s.blocks = blocks_;
     return s;
   }
 
  private:
+  struct Block {
+    Block* next;
+    size_t size;
+  };
+
+  ABSL_ATTRIBUTE_NOINLINE void AllocSlow(size_t bytes, size_t align)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(arena_lock_);
+  void StashRemainingFreeArea() ABSL_EXCLUSIVE_LOCKS_REQUIRED(arena_lock_);
+  Block* TryPopFromFreelist(size_t bytes, size_t align)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(arena_lock_);
+
   // How much to allocate from system at a time
   static constexpr int kAllocIncrement = 128 << 10;
+  // Maximum number of blocks to keep on freelist_.
+  static constexpr int kMaxFreelistBlocks = 100;
 
   mutable absl::base_internal::SpinLock arena_lock_{
       absl::base_internal::SCHEDULE_KERNEL_ONLY};
 
   // Free area from which to carve new objects
   char* free_area_ ABSL_GUARDED_BY(arena_lock_) = nullptr;
   size_t free_avail_ ABSL_GUARDED_BY(arena_lock_) = 0;
+  Block* freelist_ ABSL_GUARDED_BY(arena_lock_) = nullptr;
 
   // Total number of bytes allocated from this arena
   size_t bytes_allocated_ ABSL_GUARDED_BY(arena_lock_) = 0;
@@ -103,7 +120,11 @@ class ABSL_CACHELINE_ALIGNED Arena {
   // that these bytes are disjoint from the ones counted in `bytes_allocated`.
   size_t bytes_nonresident_ ABSL_GUARDED_BY(arena_lock_) = 0;
   // Total number of blocks/free areas managed by this Arena.
-  size_t blocks_ ABSL_GUARDED_BY(arena_lock_) = 0;
+  uint16_t blocks_ ABSL_GUARDED_BY(arena_lock_) = 0;
+  // Total number of blocks on the freelist. Capped at kMaxFreelistBlocks.
+  uint8_t freelist_blocks_ ABSL_GUARDED_BY(arena_lock_) = 0;
+  // Total number of bytes on the freelist.
+  uint32_t freelist_bytes_unallocated_ ABSL_GUARDED_BY(arena_lock_) = 0;
 
   Arena(const Arena&) = delete;
   Arena& operator=(const Arena&) = delete;