diff --git a/tcmalloc/cpu_cache.h b/tcmalloc/cpu_cache.h index 3074e797a..1b81a84c6 100644 --- a/tcmalloc/cpu_cache.h +++ b/tcmalloc/cpu_cache.h @@ -466,6 +466,15 @@ class CpuCache { // the slab based on miss-counts and resizes if so. void ResizeSlabIfNeeded(); + // Resizes the slab for a single if needed based on its miss-counts. + void ResizeCpuSlabIfNeeded(int cpu); + DynamicSlabResize ShouldResizeCpuSlab(int cpu); + uint8_t GetCpuSlabShift(int cpu) const { + uint8_t s = cpu_shift_[cpu].load(std::memory_order_relaxed); + if (s != 0) return s; + return freelist_.GetShift(); + } + // Reports total cache underflows and overflows for . CpuCacheMissStats GetTotalCacheMissStats(int cpu) const; @@ -760,6 +769,15 @@ class CpuCache { // ResizeSlabs. This memory is allocated on the arena, and it is nonresident // while not in use. void* slabs_by_shift_[kTotalPossibleSlabs] = {nullptr}; + + // For per-cpu asymmetric slab sizing prototype: + // Tracks current shift per cpu. + std::atomic cpu_shift_[kMaxCpus] = {0}; + // Standalone slab allocations per cpu. + std::atomic cpu_slab_base_[kMaxCpus] = {nullptr}; + // Reusable standalone slab allocations per cpu and shift offset: + std::atomic cpu_slabs_by_shift_[kMaxCpus][kNumPossiblePerCpuShifts] = + {}; }; template @@ -992,6 +1010,9 @@ inline void CpuCache::Activate() { TC_CHECK_LE(per_cpu_shift, shift_bounds_.max_shift); TC_CHECK_EQ(shift_bounds_.max_shift - shift_bounds_.initial_shift + 1, kNumPossiblePerCpuShifts); + for (int cpu = 0; cpu < num_cpus; ++cpu) { + cpu_shift_[cpu].store(per_cpu_shift, std::memory_order_relaxed); + } // Deal with size classes that correspond only to partitions that are in // use. If NUMA awareness and security partitions are disabled then we may @@ -1596,6 +1617,9 @@ void CpuCache::ResizeSizeClassMaxCapacities() ShiftOffset(per_cpu_shift, shift_bounds_.initial_shift), new_resize_slab_offset); + for (int cpu = 0; cpu < num_cpus; ++cpu) { + freelist_.UnbindCpuSlab(cpu, DrainHandler{*this, nullptr}); + } info = freelist_.UpdateMaxCapacities( new_slabs, GetShiftMaxCapacity{max_capacity_, per_cpu_shift, shift_bounds_}, @@ -1686,7 +1710,7 @@ void CpuCache::ResizeCpuSizeClasses(int cpu) { { AllocationGuardSpinLockHolder h(resize_[cpu].lock); subtle::percpu::ScopedSlabCpuStop cpu_stop(freelist_, cpu); - const auto max_capacity = GetMaxCapacityFunctor(freelist_.GetShift()); + const auto max_capacity = GetMaxCapacityFunctor(freelist_.GetCpuShift(cpu)); size_t size_classes_to_resize = 5; TC_ASSERT_LT(size_classes_to_resize, kNumClasses); for (size_t i = 0; i < size_classes_to_resize; ++i) { @@ -2333,56 +2357,167 @@ void CpuCache::ResizeSlabIfNeeded() ABSL_NO_THREAD_SAFETY_ANALYSIS { const DynamicSlabResize resize = ShouldResizeSlab(); if (resize == DynamicSlabResize::kGrow) { - if (per_cpu_shift == shift_bounds_.max_shift) return; + if (per_cpu_shift == shift_bounds_.max_shift) goto percpu_resize; ++per_cpu_shift; dynamic_slab_info_ .grow_count[ShiftOffset(per_cpu_shift, shift_bounds_.initial_shift)] .fetch_add(1, std::memory_order_relaxed); } else if (resize == DynamicSlabResize::kShrink) { - if (per_cpu_shift == shift_bounds_.initial_shift) return; + if (per_cpu_shift == shift_bounds_.initial_shift) goto percpu_resize; --per_cpu_shift; dynamic_slab_info_ .shrink_count[ShiftOffset(per_cpu_shift, shift_bounds_.initial_shift)] .fetch_add(1, std::memory_order_relaxed); + } else { + goto percpu_resize; + } + + { + const auto new_shift = subtle::percpu::ToShiftType(per_cpu_shift); + const int64_t new_slabs_size = + subtle::percpu::GetSlabsAllocSize(new_shift, num_cpus); + forwarder_.ArenaUpdateAllocatedAndNonresident(new_slabs_size, 0); + + for (int cpu = 0; cpu < num_cpus; ++cpu) resize_[cpu].lock.lock(); + ResizeSlabsInfo info; + const uint8_t resize_offset = + resize_slab_offset_.load(std::memory_order_relaxed); + int64_t reused_bytes; + { + AllocationGuard enforce_no_alloc; + + void* new_slabs; + std::tie(new_slabs, reused_bytes) = AllocOrReuseSlabs( + [&](size_t size, std::align_val_t align) { + return forwarder_.AllocReportedImpending(size, align); + }, + new_shift, num_cpus, + ShiftOffset(per_cpu_shift, shift_bounds_.initial_shift), + resize_offset); + for (int cpu = 0; cpu < num_cpus; ++cpu) { + freelist_.UnbindCpuSlab(cpu, DrainHandler{*this, nullptr}); + } + info = freelist_.ResizeSlabs( + new_shift, new_slabs, + GetShiftMaxCapacity{max_capacity_, per_cpu_shift, shift_bounds_}, + [this](int cpu) { return HasPopulated(cpu); }, + DrainHandler{*this, nullptr}); + } + for (int cpu = 0; cpu < num_cpus; ++cpu) resize_[cpu].lock.unlock(); + + MadviseAwaySlabs(info.old_slabs, info.old_slabs_size); + const int64_t old_slabs_size = info.old_slabs_size; + forwarder_.ArenaUpdateAllocatedAndNonresident( + -old_slabs_size, old_slabs_size - reused_bytes); + } + +percpu_resize: + if (forwarder_.per_cpu_caches_dynamic_slab_enabled()) { + for (int cpu = 0; cpu < num_cpus; ++cpu) { + if (HasPopulated(cpu)) { + ResizeCpuSlabIfNeeded(cpu); + } + } + } +} + +template +inline typename CpuCache::DynamicSlabResize +CpuCache::ShouldResizeCpuSlab(int cpu) { + CpuCacheMissStats misses = + GetAndUpdateIntervalCacheMissStats(cpu, MissCount::kSlabResize); + + if (misses.overflows == 0 && misses.underflows == 0) { + uint8_t current_shift = cpu_shift_[cpu].load(std::memory_order_relaxed); + if (current_shift > shift_bounds_.initial_shift) { + return DynamicSlabResize::kShrink; + } + return DynamicSlabResize::kNoop; + } + + if (misses.overflows + 1 > + (misses.underflows + 1) * + forwarder_.per_cpu_caches_dynamic_slab_grow_threshold()) { + return DynamicSlabResize::kGrow; + } else if (misses.overflows < + misses.underflows * + forwarder_.per_cpu_caches_dynamic_slab_shrink_threshold()) { + return DynamicSlabResize::kShrink; + } + + return DynamicSlabResize::kNoop; +} + +template +void CpuCache::ResizeCpuSlabIfNeeded(int cpu) + ABSL_NO_THREAD_SAFETY_ANALYSIS { + if (!forwarder_.per_cpu_caches_dynamic_slab_enabled()) return; + + uint8_t current_shift = cpu_shift_[cpu].load(std::memory_order_relaxed); + if (current_shift == 0) { + current_shift = shift_bounds_.initial_shift; + } + + const DynamicSlabResize resize = ShouldResizeCpuSlab(cpu); + uint8_t new_shift = current_shift; + + if (resize == DynamicSlabResize::kGrow) { + if (current_shift >= shift_bounds_.max_shift) return; + new_shift = current_shift + 1; + } else if (resize == DynamicSlabResize::kShrink) { + if (current_shift <= shift_bounds_.initial_shift) return; + new_shift = current_shift - 1; } else { return; } - const auto new_shift = subtle::percpu::ToShiftType(per_cpu_shift); - const int64_t new_slabs_size = - subtle::percpu::GetSlabsAllocSize(new_shift, num_cpus); - // Account for impending allocation/reusing of new slab so that we can avoid - // going over memory limit. - forwarder_.ArenaUpdateAllocatedAndNonresident(new_slabs_size, 0); + const auto shift_type = subtle::percpu::ToShiftType(new_shift); + const size_t slab_size = subtle::percpu::GetSlabsAllocSize(shift_type, 1); - for (int cpu = 0; cpu < num_cpus; ++cpu) resize_[cpu].lock.lock(); - ResizeSlabsInfo info; - const uint8_t resize_offset = - resize_slab_offset_.load(std::memory_order_relaxed); - int64_t reused_bytes; + const uint8_t shift_offset = + ShiftOffset(new_shift, shift_bounds_.initial_shift); + void* new_slab = + cpu_slabs_by_shift_[cpu][shift_offset].load(std::memory_order_relaxed); + if (new_slab == nullptr) { + new_slab = forwarder_.Alloc(slab_size, subtle::percpu::kPhysicalPageAlign); + if (new_slab == nullptr) return; + ANNOTATE_MEMORY_IS_INITIALIZED(new_slab, slab_size); + cpu_slabs_by_shift_[cpu][shift_offset].store(new_slab, + std::memory_order_relaxed); + } else { + ErrnoRestorer errno_restorer; + madvise(new_slab, slab_size, MADV_HUGEPAGE); + } + + void* old_slab = nullptr; + size_t old_slab_size = 0; { - // We can't allocate while holding the per-cpu spinlocks. AllocationGuard enforce_no_alloc; + resize_[cpu].lock.lock(); - void* new_slabs; - std::tie(new_slabs, reused_bytes) = AllocOrReuseSlabs( + old_slab = cpu_slab_base_[cpu].load(std::memory_order_relaxed); + if (old_slab != nullptr) { + old_slab_size = subtle::percpu::GetSlabsAllocSize( + subtle::percpu::ToShiftType(current_shift), 1); + } + + freelist_.RebindCpuSlab( [&](size_t size, std::align_val_t align) { - return forwarder_.AllocReportedImpending(size, align); + return forwarder_.Alloc(size, align); }, - new_shift, num_cpus, - ShiftOffset(per_cpu_shift, shift_bounds_.initial_shift), resize_offset); - info = freelist_.ResizeSlabs( - new_shift, new_slabs, - GetShiftMaxCapacity{max_capacity_, per_cpu_shift, shift_bounds_}, - [this](int cpu) { return HasPopulated(cpu); }, + cpu, new_slab, shift_type, + GetShiftMaxCapacity{max_capacity_, new_shift, shift_bounds_}, DrainHandler{*this, nullptr}); + + cpu_shift_[cpu].store(new_shift, std::memory_order_relaxed); + cpu_slab_base_[cpu].store(new_slab, std::memory_order_release); + + resize_[cpu].lock.unlock(); } - for (int cpu = 0; cpu < num_cpus; ++cpu) resize_[cpu].lock.unlock(); - MadviseAwaySlabs(info.old_slabs, info.old_slabs_size); - const int64_t old_slabs_size = info.old_slabs_size; - forwarder_.ArenaUpdateAllocatedAndNonresident(-old_slabs_size, - old_slabs_size - reused_bytes); + if (old_slab != nullptr) { + MadviseAwaySlabs(old_slab, old_slab_size); + } } template @@ -2558,11 +2693,14 @@ inline void CpuCache::Print(Printer& out) const { uint64_t rbytes = UsedBytes(cpu); bool populated = HasPopulated(cpu); uint64_t unallocated = Unallocated(cpu); + uint8_t shift = freelist_.GetCpuShift(cpu); + size_t slab_size = subtle::percpu::GetSlabsAllocSize( + subtle::percpu::ToShiftType(shift), 1); out.printf( "cpu %3d: %12u" " bytes (%7.1f MiB) with" - "%12u bytes unallocated %s%s\n", - cpu, rbytes, rbytes / MiB, unallocated, + "%12u bytes unallocated, slab size %10zu bytes %s%s\n", + cpu, rbytes, rbytes / MiB, unallocated, slab_size, allowed_cpus.IsSet(cpu) ? " active" : "", populated ? " populated" : ""); } diff --git a/tcmalloc/cpu_cache_test.cc b/tcmalloc/cpu_cache_test.cc index 6502ae21e..c57eeb0dd 100644 --- a/tcmalloc/cpu_cache_test.cc +++ b/tcmalloc/cpu_cache_test.cc @@ -1247,6 +1247,59 @@ TEST_F(DynamicWideSlabTest, DynamicSlabThreshold) { EXPECT_EQ(CpuCachePeer::GetSlabShift(cache), shift + 1); } +TEST_F(DynamicWideSlabTest, PerCpuAsymmetricSlabResizing) { + if (!subtle::percpu::IsFast()) { + return; + } + constexpr double kDynamicSlabGrowThreshold = 0.9; + CpuCache cache; + TestStaticForwarder& forwarder = cache.forwarder(); + forwarder.dynamic_slab_enabled_ = true; + forwarder.dynamic_slab_grow_threshold_ = kDynamicSlabGrowThreshold; + SizeMap size_map; + ASSERT_TRUE(size_map.Init(size_map.CurrentClasses().classes)); + forwarder.size_map_ = size_map; + + cache.Activate(); + + constexpr int kCpuId0 = 0; + constexpr int kCpuId1 = 1; + + // Accumulate overflows and underflows for kCpuId0 (Hot CPU). + HotCacheOperations(cache, kCpuId0); + CpuCache::CpuCacheMissStats interval_misses0 = + cache.GetIntervalCacheMissStats(kCpuId0, MissCount::kSlabResize); + ASSERT_GT(interval_misses0.overflows, + interval_misses0.underflows * kDynamicSlabGrowThreshold); + + // Perform cold operations on kCpuId1 (Cold CPU). + for (int i = 0; i < 1024; ++i) { + ColdCacheOperations(cache, kCpuId1, /*size_class=*/1); + cache.Reclaim(kCpuId1); + } + + cpu_cache_internal::SlabShiftBounds shift_bounds = + cache.GetPerCpuSlabShiftBounds(); + const int initial_shift = shift_bounds.initial_shift; + + EXPECT_EQ(cache.GetCpuSlabShift(kCpuId0), initial_shift); + EXPECT_EQ(cache.GetCpuSlabShift(kCpuId1), initial_shift); + + // Perform per-CPU asymmetric resizing! + cache.ResizeCpuSlabIfNeeded(kCpuId0); + cache.ResizeCpuSlabIfNeeded(kCpuId1); + + // Hot CPU 0 should have grown independently to initial_shift + 1! + EXPECT_EQ(cache.GetCpuSlabShift(kCpuId0), initial_shift + 1); + // Cold CPU 1 should stay at initial_shift! + EXPECT_EQ(cache.GetCpuSlabShift(kCpuId1), initial_shift); + + // When CPU 0 becomes idle (0 overflows, 0 underflows), calling + // ResizeCpuSlabIfNeeded should demote it back down to initial_shift. + cache.ResizeCpuSlabIfNeeded(kCpuId0); + EXPECT_EQ(cache.GetCpuSlabShift(kCpuId0), initial_shift); +} + // Test that when dynamic slab parameters change, things still work. TEST_F(DynamicWideSlabTest, DynamicSlabParamsChange) { if (!subtle::percpu::IsFast()) { @@ -1866,6 +1919,7 @@ TEST(CpuCacheTest, Fuzz) { Printer p(mallocz.data(), mallocz.size()); env.cache().Print(p); std::cout << mallocz; + EXPECT_THAT(mallocz, testing::HasSubstr("slab size")); } // TODO(b/179516472): Enable this test. diff --git a/tcmalloc/internal/percpu_tcmalloc.h b/tcmalloc/internal/percpu_tcmalloc.h index 056cacaee..319cbb644 100644 --- a/tcmalloc/internal/percpu_tcmalloc.h +++ b/tcmalloc/internal/percpu_tcmalloc.h @@ -141,6 +141,14 @@ class TcmallocSlab { // other than Push/Pop/PushBatch/PopBatch are invalid. void InitCpu(int cpu, absl::FunctionRef capacity); + // Rebinds to a standalone slab buffer of size corresponding to + // . + void RebindCpuSlab(absl::FunctionRef alloc, + int cpu, void* slab, Shift shift, + absl::FunctionRef capacity, + DrainHandler drain_handler); + void UnbindCpuSlab(int cpu, DrainHandler drain_handler); + // Update maximum capacities allocated to each size class. // Build and initialize so as to use new maximum capacities // provided by callback for the . @@ -290,6 +298,29 @@ class TcmallocSlab { return ToUint8(GetSlabsAndShift(std::memory_order_relaxed).second); } + uint8_t GetCpuShift(int cpu) const { + if (!state_.empty() && cpu >= 0) { + uint8_t s = state_[cpu].shift.load(std::memory_order_relaxed); + if (s != 0) return s; + } + return GetShift(); + } + + const std::atomic* GetBeginPtr(int cpu, size_t size_class) const { + uint8_t shift = GetCpuShift(cpu); + if (!state_.empty() && shift != GetShift() && shift >= 14 && shift <= 18) { + uint8_t idx = shift - 14; + if (begins_by_shift_[idx] != nullptr) { + return &begins_by_shift_[idx][size_class]; + } + } + return &begins_[size_class]; + } + + uint16_t GetBegin(int cpu, size_t size_class) const { + return GetBeginPtr(cpu, size_class)->load(std::memory_order_relaxed); + } + constexpr static size_t GetCpuStateSize() { return sizeof(CpuState); } private: @@ -361,9 +392,9 @@ class TcmallocSlab { return slabs_and_shift_.load(order).Get(); } - static void* CpuMemoryStart(void* slabs, Shift shift, int cpu); - static AtomicHeader* GetHeader(void* slabs, Shift shift, int cpu, - size_t size_class); + void* CpuMemoryStart(void* slabs, Shift shift, int cpu) const; + AtomicHeader* GetHeader(void* slabs, Shift shift, int cpu, + size_t size_class) const; static Header LoadHeader(AtomicHeader* hdrp); static void StoreHeader(AtomicHeader* hdrp, Header hdr); void DrainCpu(void* slabs, Shift shift, int cpu, DrainHandler drain_handler); @@ -387,11 +418,20 @@ class TcmallocSlab { // Remote Cpu operation (Resize/Drain/Grow/Shrink) is running so any local // operations (Push/Pop) should fail. std::atomic stopped{false}; + // Per-CPU standalone slab base pointer for asymmetric slab sizing + // prototype. + std::atomic slab_base{nullptr}; + // Per-CPU shift for asymmetric slab sizing prototype. + std::atomic shift{0}; }; absl::Span state_ = {}; // begins_[size_class] is offset of the size_class region in the slabs area. std::atomic* begins_ = nullptr; + // For asymmetric per-CPU slab sizing prototype: + // Stores size-class begin offsets for each possible shift (indices 0..4 for + // shifts 14..18). + std::atomic* begins_by_shift_[5] = {nullptr}; }; // RAII for StopCpu/StartCpu. @@ -418,7 +458,7 @@ inline size_t TcmallocSlab::Length(int cpu, size_t size_class) const { const auto [slabs, shift] = GetSlabsAndShift(std::memory_order_relaxed); Header hdr = LoadHeader(GetHeader(slabs, shift, cpu, size_class)); - uint16_t begin = begins_[size_class].load(std::memory_order_relaxed); + uint16_t begin = GetBegin(cpu, size_class); // We can read inconsistent hdr/begin during Resize, to avoid surprising // callers return 0 instead of overflows values. return std::max(0, hdr.current - begin); @@ -429,7 +469,7 @@ inline size_t TcmallocSlab::Capacity(int cpu, size_t size_class) const { const auto [slabs, shift] = GetSlabsAndShift(std::memory_order_relaxed); Header hdr = LoadHeader(GetHeader(slabs, shift, cpu, size_class)); - uint16_t begin = begins_[size_class].load(std::memory_order_relaxed); + uint16_t begin = GetBegin(cpu, size_class); return std::max(0, hdr.end - begin); } @@ -981,10 +1021,11 @@ inline size_t TcmallocSlab::Grow( int cpu, size_t size_class, size_t len, absl::FunctionRef max_capacity) { const auto [slabs, shift] = GetSlabsAndShift(std::memory_order_relaxed); - const size_t max_cap = max_capacity(ToUint8(shift)); + const uint8_t cpu_shift = GetCpuShift(cpu); + const size_t max_cap = max_capacity(cpu_shift); auto* hdrp = GetHeader(slabs, shift, cpu, size_class); Header hdr = LoadHeader(hdrp); - uint16_t begin = begins_[size_class].load(std::memory_order_relaxed); + uint16_t begin = GetBegin(cpu, size_class); ssize_t have = static_cast(max_cap - (hdr.end - begin)); if (have <= 0) { return 0; @@ -1032,8 +1073,10 @@ inline size_t TcmallocSlab::PopBatch(size_t size_class, void** batch, size_t len) { TC_ASSERT_NE(size_class, 0); TC_ASSERT_NE(len, 0); - const size_t n = TcmallocSlab_Internal_PopBatch(size_class, batch, len, - &begins_[size_class]); + const size_t n = TcmallocSlab_Internal_PopBatch( + size_class, batch, len, + const_cast*>( + GetBeginPtr(VirtualCpu::get(), size_class))); TC_ASSERT_LE(n, len); // PopBatch is implemented in assembly, msan does not know that the returned @@ -1045,13 +1088,20 @@ inline size_t TcmallocSlab::PopBatch(size_t size_class, template inline void* TcmallocSlab::CpuMemoryStart(void* slabs, Shift shift, - int cpu) { + int cpu) const { + if (!state_.empty()) { + void* per_cpu_base = state_[cpu].slab_base.load(std::memory_order_relaxed); + if (per_cpu_base != nullptr) { + return per_cpu_base; + } + } return &static_cast(slabs)[cpu << ToUint8(shift)]; } template inline auto TcmallocSlab::GetHeader(void* slabs, Shift shift, - int cpu, size_t size_class) + int cpu, + size_t size_class) const -> AtomicHeader* { TC_ASSERT_NE(size_class, 0); return &static_cast( @@ -1123,6 +1173,58 @@ void TcmallocSlab::InitCpu( InitCpuImpl(slabs, shift, cpu, capacity); } +template +void TcmallocSlab::RebindCpuSlab( + absl::FunctionRef alloc, int cpu, + void* slab, Shift shift, absl::FunctionRef capacity, + DrainHandler drain_handler) { + ScopedSlabCpuStop cpu_stop(*this, cpu); + const auto [old_slabs, old_shift] = + GetSlabsAndShift(std::memory_order_relaxed); + DrainCpu(old_slabs, old_shift, cpu, drain_handler); + + state_[cpu].slab_base.store(slab, std::memory_order_relaxed); + state_[cpu].shift.store(ToUint8(shift), std::memory_order_relaxed); + + uint8_t idx = ToUint8(shift) - 14; + if (idx < 5 && begins_by_shift_[idx] == nullptr) { + auto* new_begins = static_cast*>( + alloc(sizeof(std::atomic) * NumClasses, + std::align_val_t{ABSL_CACHELINE_SIZE})); + size_t consumed_bytes = (NumClasses * sizeof(Header) + sizeof(void*) - 1) & + ~(sizeof(void*) - 1); + bool prev_empty = false; + for (size_t size_class = 1; size_class < NumClasses; ++size_class) { + size_t cap = capacity(size_class); + if (!prev_empty) { + consumed_bytes += sizeof(void*); + } + prev_empty = cap == 0; + new_begins[size_class].store(consumed_bytes / sizeof(void*), + std::memory_order_relaxed); + consumed_bytes += cap * sizeof(void*); + } + begins_by_shift_[idx] = new_begins; + } + + InitCpuImpl(slab, shift, cpu, capacity); +} + +template +void TcmallocSlab::UnbindCpuSlab(int cpu, + DrainHandler drain_handler) { + if (state_[cpu].slab_base.load(std::memory_order_relaxed) == nullptr) { + return; + } + ScopedSlabCpuStop cpu_stop(*this, cpu); + const auto [old_slabs, old_shift] = + GetSlabsAndShift(std::memory_order_relaxed); + DrainCpu(old_slabs, old_shift, cpu, drain_handler); + + state_[cpu].slab_base.store(nullptr, std::memory_order_relaxed); + state_[cpu].shift.store(0, std::memory_order_relaxed); +} + template void TcmallocSlab::InitCpuImpl( void* slabs, Shift shift, int cpu, @@ -1232,7 +1334,7 @@ void TcmallocSlab::DrainCpu(void* slabs, Shift shift, int cpu, DrainHandler drain_handler) { TC_ASSERT(state_[cpu].stopped.load(std::memory_order_relaxed)); for (size_t size_class = 1; size_class < NumClasses; ++size_class) { - uint16_t begin = begins_[size_class].load(std::memory_order_relaxed); + uint16_t begin = GetBegin(cpu, size_class); auto* hdrp = GetHeader(slabs, shift, cpu, size_class); Header hdr = LoadHeader(hdrp); if (hdr.current == 0) { @@ -1404,10 +1506,10 @@ size_t TcmallocSlab::GrowOtherCache( absl::FunctionRef max_capacity) { TC_ASSERT(state_[cpu].stopped.load(std::memory_order_relaxed)); const auto [slabs, shift] = GetSlabsAndShift(std::memory_order_relaxed); - const size_t max_cap = max_capacity(ToUint8(shift)); + const size_t max_cap = max_capacity(GetCpuShift(cpu)); auto* hdrp = GetHeader(slabs, shift, cpu, size_class); Header hdr = LoadHeader(hdrp); - uint16_t begin = begins_[size_class].load(std::memory_order_relaxed); + uint16_t begin = GetBegin(cpu, size_class); uint16_t to_grow = std::min(len, max_cap - (hdr.end - begin)); hdr.end += to_grow; StoreHeader(hdrp, hdr); @@ -1427,7 +1529,7 @@ size_t TcmallocSlab::ShrinkOtherCache( // the list first to create enough capacity that can be shrunk. // If we pop items, we also execute callbacks. const uint16_t unused = hdr.end - hdr.current; - uint16_t begin = begins_[size_class].load(std::memory_order_relaxed); + uint16_t begin = GetBegin(cpu, size_class); if (unused < len && hdr.current != begin) { uint16_t pop = std::min(len - unused, hdr.current - begin); void** batch = reinterpret_cast(CpuMemoryStart(slabs, shift, cpu)) +